{
"measurement": {
"model.layers.0.self_attn": [
{
"accuracy": 0.988181887782718,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905226622512074,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908563015051186,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995120972836096,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953012601594979,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953221018475137,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957821328364509,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958086311449542,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961171420903778,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996390018844977,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975475614253235,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997689324712969,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976999293032446,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978594946564714,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986912820022553,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988454701011314,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988300505088104,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999010514486336,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995097647800943,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.0.mlp": [
{
"accuracy": 0.9792020290991977,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800228972085997,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828391120965151,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846248017182868,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921064477928571,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927255188507077,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937175056088323,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947061924214818,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.99553404007058,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959812763845548,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965231383256125,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977489991732383,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980529512517693,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982968409037168,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998376751122506,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985968058769542,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986407154953524,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.self_attn": [
{
"accuracy": 0.9672311684606891,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719883471139168,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737645165112457,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985823112637981,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862722542725111,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985587070549005,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892922459954494,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.989316458960897,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899983597968361,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905817168519685,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925705408922544,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929946398999738,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993348806581803,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938122166418716,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963267335030985,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968363114779717,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970493720795371,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997773156960887,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991369583999345,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.mlp": [
{
"accuracy": 0.9646863441326117,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9653634494660717,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753429945068139,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785811676594772,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845928379187459,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855749832074109,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897734847556996,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918818807866621,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992735427142562,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923408102048071,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932004434889868,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960494164352942,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965715376709244,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978031538684216,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979688742406372,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988562632413385,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992796366446113,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.self_attn": [
{
"accuracy": 0.9550868998232641,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9580719310202097,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9608700065628478,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771566798812464,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9772267655322426,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792278492333073,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830960968607351,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855880696130427,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866164146285308,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870820457213804,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897988244008861,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990430913252854,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990587528500902,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911730362494525,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942964204636059,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954497426576716,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956306264931826,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997543984840281,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988480258371534,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.mlp": [
{
"accuracy": 0.9706852704678711,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735651047116047,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771981777525262,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781231449328756,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987274305698903,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880576996426833,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895840635596725,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915399621853507,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922476739524618,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939753245562315,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928767331263149,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962148560872791,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934811412822455,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967923270948966,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972834402701425,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974684334929885,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974460165678082,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.self_attn": [
{
"accuracy": 0.9785212972446492,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9790413024786272,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9815082883854446,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852008549251446,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860690127028838,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985972274241871,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871734517969584,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870501662310409,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900290845452171,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911167869025743,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929646466687125,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946095760372516,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931447616729298,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951180108320458,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971480861269427,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976346530387864,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974649604961374,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998896127756636,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992010938294698,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.mlp": [
{
"accuracy": 0.9751616747755754,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757040434173847,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816429854958857,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833331604145075,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884365927918177,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892253765934392,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916727296444342,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994228563173429,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947878412510219,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943514370334972,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950104233208358,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970811460853407,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974715429002812,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984456257968161,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985101138192573,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989705082215982,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995744086711631,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.self_attn": [
{
"accuracy": 0.9779373817635995,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786406739271785,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.977964596283671,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807850101129397,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9815308306072104,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860788320779408,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825658450932487,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874572642392626,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900140193609619,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902741286756569,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937829615940389,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954309798838374,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953344654426665,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957941107660238,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946163789075064,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975970885256844,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947756019567973,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988960997541557,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988358689588495,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.mlp": [
{
"accuracy": 0.9688971587211678,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9696191437052268,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9761930420798691,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781178909500963,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854367524630537,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864100560692972,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891929401231832,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927552973508442,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934334021772405,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929176327821455,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937367661807098,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963274242809826,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968150832481977,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980393259473587,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998128776502869,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998647139340296,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994695583583578,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.self_attn": [
{
"accuracy": 0.9934854681334017,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936048768117631,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941565643530339,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944627043962675,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980530420281483,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980225038200029,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982732692855949,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998234047634086,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983269924077353,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986872711451724,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989055259533128,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99886210472323,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989174995387561,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992166433887752,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994914237805029,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995935993073018,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995395619296591,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997666416456923,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998293243177039,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.mlp": [
{
"accuracy": 0.994445853927908,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946441971329286,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955737016678444,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958366815821806,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975226803492804,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976910994951859,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980743121214848,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998732751596866,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988492350859911,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987835881306397,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989245435618183,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993575843430957,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994412814305275,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996317273854449,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996541458178974,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997269194565505,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998600127314843,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.self_attn": [
{
"accuracy": 0.9904892652932751,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906409585750416,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921368038722951,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932639473362973,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950121310662086,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995024869488062,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955953923421667,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956083014288819,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958337758373665,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958892014416817,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979068680151709,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979650487167466,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980195732542166,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981301347173652,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989364158153828,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998938314512519,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990617946543927,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993135417339785,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997128229465728,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.mlp": [
{
"accuracy": 0.9846501943753346,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850082801360833,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881914549899337,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891515928320587,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923110673400132,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930417877131779,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942450737458115,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.995800440689843,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963843910832351,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958698258453392,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965227922289878,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975492957133898,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981683408259414,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984867597537997,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983923415910747,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991541302557667,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994301780687621,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.self_attn": [
{
"accuracy": 0.9841209074580356,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844128602723542,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870968415941063,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882984493957147,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890821484211636,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888924204891449,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894706830872517,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.989357328949202,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915383332408965,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992071475208688,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943595430568645,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995761700881351,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945256299780387,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960911562911382,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977383213607889,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998163045046998,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978369310510492,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993047430195331,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99932517923911,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.mlp": [
{
"accuracy": 0.9790346730794561,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795730401712813,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837592099303085,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849541744854498,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901776566219173,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908577753966185,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925951235682556,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950907241283474,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955594741531011,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952136594802141,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995778819925985,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975133998760659,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978476362456078,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998659663922521,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987238296338259,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990489122742714,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996201546210183,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.self_attn": [
{
"accuracy": 0.982518340755058,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830740897328054,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826282878945533,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842283199541271,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845140395735047,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892943484071446,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851284256008895,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900925251606264,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916824092475796,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919452578025428,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956393363689514,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963522318591315,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963421957800165,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966627149229967,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995526058660624,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981528510351813,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956081756145546,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990678904179827,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990365360725647,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.mlp": [
{
"accuracy": 0.972500848681911,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9731653923855016,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785328942203992,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801494167431405,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987121239427085,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879814205250066,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903089693796477,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993556470892049,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941627663609228,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937370755636182,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944630392312416,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967453053931853,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971788537590519,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.99824778296323,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983375432999119,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987773895294307,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995102941106061,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.self_attn": [
{
"accuracy": 0.9741757142994749,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9748946966015195,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773429183681545,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.980271531857158,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840958660076323,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838560229216359,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985610960786672,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852904192613143,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884337356657182,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889363897474188,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919148702296967,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937614194166503,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921663298247088,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946424139075374,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965644011788658,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973700117736467,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968670066078439,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987495580204077,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992185776400698,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.mlp": [
{
"accuracy": 0.9685024022075691,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9692565758565539,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753800290881803,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771929966089757,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852353176592212,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862037152542096,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888264113596004,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926792913067498,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933468162719357,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928431505533425,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936635622901744,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962787064968756,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967677320574263,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980066436661506,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980978654961004,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985825377547084,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994499411933295,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.self_attn": [
{
"accuracy": 0.9650325978076771,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9663777674891447,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9702068184453406,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9745990340352843,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9815328016288971,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9821085961241471,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838331947348228,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844299718786619,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869849412996126,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878730611189416,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909446675722536,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992346326550959,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913097165622994,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936373298987746,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959304983310989,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967495277217638,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963642981356794,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984118848959426,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990504456693184,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.mlp": [
{
"accuracy": 0.9640493104724508,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9648785360745693,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9717650025788891,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737753832810804,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830834597447201,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842024138197303,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871294767301726,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916342650785258,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923938453981751,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918004960605973,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927406308951935,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957340177222106,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962939118022883,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977183144319018,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978201026403296,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983613701616904,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993733453340706,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.self_attn": [
{
"accuracy": 0.9580960617841858,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.959204456719913,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9612197937737954,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9647103435310879,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9756970884964654,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755625819885417,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781024338873593,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783580843359232,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.979738743044436,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795506914195261,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880860731201736,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882005987149712,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884788296185434,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883457353387616,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949982278661704,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948593079185996,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954194071653643,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971739609517452,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986061093850845,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.mlp": [
{
"accuracy": 0.957710790006738,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9586227456793973,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9667295036151221,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690539944720896,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799521352703634,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813034099183584,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.984688613563776,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901031009901903,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909935675463394,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902785396644551,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913858224294687,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949307759954152,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955981081400654,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972902888395382,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974080902695852,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980423010542596,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992589035782179,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.self_attn": [
{
"accuracy": 0.9437818511536247,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9454803886382204,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9510315232174962,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9570236503097572,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9741042739662685,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737273048805563,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.977945087036412,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9778846279276829,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9780513137873066,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9772258698450107,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804512574582508,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785047191636342,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796748122592506,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805085556581616,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910364070613134,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911644537933171,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991571066410918,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953258764243832,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997508619630121,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.mlp": [
{
"accuracy": 0.9684471350751425,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9686308828623671,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719213557576662,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729141159669349,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858257897492302,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.986984996223136,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988453187538605,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864653166089403,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915323097405857,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.981974175886104,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.990706179192976,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953431839638046,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969063707088169,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962289059706229,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981495201661202,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984453865572026,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989906520288634,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.self_attn": [
{
"accuracy": 0.991222222703264,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914735040235284,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930303696739046,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946715628336135,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958194670953641,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958156190910622,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971970849682724,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971566992615791,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997443314433392,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978054684746128,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997932169886649,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981146573248368,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982650060218906,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99846255164716,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989996876622746,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992343833406554,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992061771366647,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995914469434769,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997862655321736,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.mlp": [
{
"accuracy": 0.9882108007410639,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.988525712519492,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905634407248152,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911739601704636,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944508223377756,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948174056450003,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956956595721606,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972444403438682,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974914859600463,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997316060397823,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976240778280618,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985971307553547,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987815420578332,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992407540297503,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992787128384838,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999440945055348,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997817364407973,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.self_attn": [
{
"accuracy": 0.9873969254427051,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878494029766635,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989873807117539,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927949823537155,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933975471036607,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938625438065317,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952139693655466,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959494530002734,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963406894331504,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966126861062979,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970161500264352,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973051693707117,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975529795782151,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977478381356617,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984081132509011,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988453504414984,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986788282526264,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994188220313701,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996167711575062,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.mlp": [
{
"accuracy": 0.9861857591343945,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865251570498865,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890174675781868,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897519880905747,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934859367969789,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939257934763047,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949765705969185,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967587351749995,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970577235860554,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968393506224904,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972084179508353,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983503546130745,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985696853438736,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991074965632594,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991536341212371,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993516700449522,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997458964085366,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.self_attn": [
{
"accuracy": 0.9833338617984402,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840504426117006,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.986159539340358,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887729272932598,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914557134095383,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921642876671333,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936549623227237,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99447121018985,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954202224402443,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957007725085867,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959890131936654,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965125766925906,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966729270649681,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997250924328048,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979658431777927,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986408064295048,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982558181877003,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993082424641072,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995115046438418,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.mlp": [
{
"accuracy": 0.9833643478096316,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837524849235227,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868780431141587,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877708949589807,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992116382835727,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926467525258073,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939458868221233,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960981214320973,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964461237633306,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961728361851879,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966117967116205,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980004357333344,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982636235041642,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998925107834542,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989736251208294,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992185026425623,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996952339731108,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.self_attn": [
{
"accuracy": 0.9777416164349568,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784466253691598,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810905197243157,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847670540115551,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891871019560647,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896522898374027,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920364440002135,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992791063519881,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933143033964658,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994281944230591,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933562289286209,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942194883907703,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941074960791555,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957705638003781,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970590344225791,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978593761763094,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974515954351151,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989821353106221,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992384601791855,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.mlp": [
{
"accuracy": 0.9842737725641775,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842989170237592,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866396800712928,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872986256439042,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991716850867593,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919980791103291,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928256753636034,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949465262409496,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944651704281569,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955754987711675,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947987596112254,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967680062951618,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981791926118987,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973557548058268,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987062712806571,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988615566919754,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991237275290156,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.self_attn": [
{
"accuracy": 0.9931017673255778,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934854618347201,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947533665077859,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958307036737862,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969689383931262,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969809268736035,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998103804678019,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980884806657406,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982190404399788,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984321636025255,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985260185540507,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985721160161407,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986709591390345,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988708682618668,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992909713963807,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994460654507795,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994496556801574,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996963847898817,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9998487072798047,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.mlp": [
{
"accuracy": 0.9838611614694329,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842227923830873,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869897388853133,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877608105491259,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922140746647002,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927399237640202,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938533939543719,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960944064476184,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964108383099205,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962362606433759,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966691668807087,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998027433696399,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998266593035384,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989351532268828,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989790826634577,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999183440414,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996788838049581,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.self_attn": [
{
"accuracy": 0.9937311708559528,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993950291316172,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947436059531021,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959479568428115,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964723477789544,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965925721672216,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972067578132019,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973632022738457,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974962172196492,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978307871650415,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982356464667981,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984129148682481,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984148351849351,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985728599376192,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989258953658129,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991917968742966,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990228990239924,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996594094034088,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997079935917434,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.mlp": [
{
"accuracy": 0.9818921840847715,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823075202048609,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853094047621677,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861333957922301,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913194609291264,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919176489653948,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931119077863466,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957410159741381,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.996096590552234,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957910632212205,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962776458430055,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977844180692429,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980779787262068,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988027179619836,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988502828958812,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990795572674344,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996385942389392,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.self_attn": [
{
"accuracy": 0.9924158624397885,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926751376570839,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935732033520349,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950933814428649,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995853007029693,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961421380223903,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968319620544973,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971760953531453,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972926203136969,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974982837815524,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979446465394607,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981026460330835,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981182084271782,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982964531334705,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988720958905393,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990924253100897,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990353085878795,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995900150587611,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997190454374313,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.mlp": [
{
"accuracy": 0.9809374475949689,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813801495260314,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844474638076989,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853042034236225,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908650474083659,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914832091522648,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927268621732334,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954973860202652,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958985837017137,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955884293150646,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960850754777264,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976842263284581,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979838381159903,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987527128848198,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988103613110357,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990454052335975,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996656989134959,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.self_attn": [
{
"accuracy": 0.9906517772837297,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910780234188822,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923771824197549,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945260066612575,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955372062854861,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956741168678395,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970433250694585,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972279310398,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974709261993044,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997689138699666,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979225759161636,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980419151497221,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982009650741116,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983612836160893,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989594124061497,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991632107994519,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999212120097486,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999544858416285,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997911093405013,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.mlp": [
{
"accuracy": 0.9789918011152431,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793626219524365,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829467537714854,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839085372360913,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990075226750617,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908277353909063,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922017055735188,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.99509747594742,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955565723418993,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.995191132931627,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957390000712812,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974440604105199,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977726569509526,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985597283934782,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986276075911806,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988768041856881,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993973256389023,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.self_attn": [
{
"accuracy": 0.9865583270837209,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870951626400807,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884486200140887,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916530945951021,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934379512713732,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935322905228915,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956370276325431,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957618185956227,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960811215906257,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964078341083797,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966449955613107,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968830770016402,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970148215989435,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972656008995775,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983080576670258,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986093430259069,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998691497110198,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992466953776066,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999638064499442,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.mlp": [
{
"accuracy": 0.978704643827912,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792064276867007,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827744099311531,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837636934152167,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898334776639546,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905011847272123,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919365530858111,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950199820910042,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954267688291637,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950849346595964,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956281243602892,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974205324940971,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977478486743119,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986151223113809,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986692147090492,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989322467437504,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996115894962139,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.self_attn": [
{
"accuracy": 0.9851279163144921,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855755017825255,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875349999699545,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906108085017064,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927301672654912,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927332852476913,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949125709551337,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948930459366622,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958379111363014,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996113524980549,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996346125140247,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967322370319284,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996742124577347,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972117704976546,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981585650055326,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984983228251191,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985501731004517,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992410264140313,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995948998349424,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.mlp": [
{
"accuracy": 0.9762413560069705,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9768079774160134,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806565529127654,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817423378362468,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.988633338288453,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893794762656877,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909394300591788,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944298871253666,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948877304130676,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945119746200937,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951225908130015,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971219166917237,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974868716672063,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984533513534667,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985201000450424,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998808562363449,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995837796833305,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.self_attn": [
{
"accuracy": 0.9852164475688416,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857511325543257,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873772486554164,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907229002810231,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926673848990744,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927387125545034,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952240903420668,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953097103188109,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958956732145069,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961171035569063,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965564171526287,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968315948070469,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969813256205893,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973026800251222,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983087002060769,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986199460568299,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988165527220668,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992505852669159,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996880750197881,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.mlp": [
{
"accuracy": 0.9738421472965887,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9744797617099002,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786611905713615,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798384745066103,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987491910027242,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883173361763751,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899999088558712,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938833779214244,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943851247846492,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939745589600582,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946493733870355,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968443640789605,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972498471125666,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983101740056698,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983842243826458,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986823202374293,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995484675937883,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.self_attn": [
{
"accuracy": 0.9840222177536864,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846544501028562,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867137368082216,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901731048563593,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922755500056634,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923288424156214,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948535569894471,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949266607015345,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952582649420947,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957058524198242,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960049473325183,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962663468537166,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962991817278395,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967903018941319,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979033105467495,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983303044787853,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983220747733292,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991566848873749,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995389122417883,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.mlp": [
{
"accuracy": 0.9722364064992258,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729199792307458,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773715478123018,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786267162447697,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986740936525166,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.987611648749168,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893978043263288,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934940504669947,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940149082532642,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935951314768509,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943012678015389,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966362457489595,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970627614039633,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981884146538121,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982698350458553,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998601310478989,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995129011268115,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.self_attn": [
{
"accuracy": 0.9818129863491968,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823933331001746,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844972173634329,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884933228604496,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910027960462398,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911745224687222,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994281976067118,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945136476798278,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946870865962026,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952790345751533,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955396510317529,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958470825620583,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960402430856208,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964265060598815,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977465470345985,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981396851975373,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99832976683913,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990272831208514,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995494018573816,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.mlp": [
{
"accuracy": 0.9708766443654895,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.971611633308624,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763593965357071,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776586355935586,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986144142691046,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870603438221702,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889028578702557,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932077526801119,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937571534813431,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933062502501631,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940526313148439,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964864923447174,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969329369977411,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981067179438756,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981903339258248,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985354495124499,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994879489930003,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.self_attn": [
{
"accuracy": 0.9806343964546135,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.981328288485345,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833932623365208,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874088429404717,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908311836127388,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908868131766978,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945006998574459,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945490600688285,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947954615283954,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956202320264358,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954744667721618,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957281486866506,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960076752621191,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965255782865968,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977307063588676,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982062418405947,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983626588119047,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989821411804051,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995895494610426,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.mlp": [
{
"accuracy": 0.9693106028594469,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.970082374044547,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749307357951215,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9762555656856612,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854093331990665,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.986385347346138,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882572574393922,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928492540861235,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993435036550325,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929543465964104,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937340060224462,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962941526900977,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967711152888736,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979874185029123,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980776511181734,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984106956490952,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993797598703837,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.self_attn": [
{
"accuracy": 0.9812108208180258,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.981872630893792,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836110673403662,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874834836901802,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904354177170286,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904952626920452,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936666743903372,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937260442805526,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938859339729932,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945326334502744,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952350776201409,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953245938093843,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956407860705727,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958596249527641,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974816004488323,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978811564873659,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980393856102111,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989542004431745,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994506897070862,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.mlp": [
{
"accuracy": 0.9683776042473159,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9691594997911077,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9741185086436177,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9754634452003398,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849051119465577,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859077478268821,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878280427502958,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926276354878945,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932193566407812,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992712958262449,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935289316590091,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961708234929431,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966618695516923,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997940985942382,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980284771943269,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983814711357761,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994464552100429,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.self_attn": [
{
"accuracy": 0.9828689206685675,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833710161772998,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985495373214546,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884718024956161,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917597079551533,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918617619887778,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943593711788324,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944789189551222,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995108576234136,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953280513566968,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953159129149035,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961527651175857,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957148240707618,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965964730240797,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977027966003669,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982122908174795,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981105735121099,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991132655306597,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999417046483263,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.mlp": [
{
"accuracy": 0.9676438235236626,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9684468280327947,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9733856735554965,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.974741205622099,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844802242066515,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855249081493208,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874579110918077,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924185410092928,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930271752129652,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925121996968397,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933457315874923,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960664350441412,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965659441169057,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978843131041351,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979751535299185,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983318340535717,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994318747745925,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.self_attn": [
{
"accuracy": 0.9814027008159381,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.982059513120667,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840644543353272,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879478776680404,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905155148651255,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904594796121513,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993484308318186,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933544744639412,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994891186808481,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995580767256845,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952864460647106,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959900832404137,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957858064199651,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965924569221801,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978688241221549,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982885164913329,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984969252162907,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990181453268681,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995843137424488,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.mlp": [
{
"accuracy": 0.9669454086964068,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9677578438269465,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.972746373154223,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9741307967587521,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984104959098132,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851741927283767,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871550514852923,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922511831391603,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928835550764281,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923462185297945,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993211616590423,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959884478563541,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965048772282898,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978514965868702,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979438389265468,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982912876140816,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994256955795112,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.self_attn": [
{
"accuracy": 0.9838767585736748,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984523262902114,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864142208948339,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899019158543333,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922076621534008,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923244699786761,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951403598595214,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995246097396471,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995558269499605,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958286302004892,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959711181306231,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961940584348907,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964123985203179,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966654340540501,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979417473261588,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982156230620794,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998381409864554,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991172925143282,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995589936473774,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.mlp": [
{
"accuracy": 0.9659944510969677,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668286560397399,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719265675858447,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9733497437187716,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836320875359601,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847260796661047,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867694221044842,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920188836802385,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926534415418772,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992100342544482,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929842830047404,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958493961450202,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963794146713457,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977701095447532,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997860862340435,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982302309516328,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993960177051026,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.self_attn": [
{
"accuracy": 0.9828102538656247,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834257486325345,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852611184806416,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888868990942443,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913771264255047,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915432943472344,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944116049691251,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945930634429189,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948917706497014,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952895131795422,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957743942075851,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960930316868287,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962092789968378,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965717097634057,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978550273165303,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982465613927496,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983650328236093,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990897305803006,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999521802726361,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.mlp": [
{
"accuracy": 0.9652672310996997,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9661148334608266,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9712913026052871,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9727510425115102,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832868703739032,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843956639647091,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864799897431543,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918448183023812,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924941181915959,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919476191954393,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928422082387107,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957697620786923,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963052452443854,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997727321056453,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997822754685522,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982019396388511,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993915333041897,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.32.self_attn": [
{
"accuracy": 0.9804228733930933,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811177626252174,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833899348925211,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877719879150391,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903624261680403,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906812847199801,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939642326269102,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944439603162831,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948235843943334,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954054031619116,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995448851945663,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957404461701548,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960290446730429,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964142020877549,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977627197268272,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981606395020591,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984422315549302,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989867683493925,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995426422882654,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.32.mlp": [
{
"accuracy": 0.9654995427515946,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9663378411511842,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9715158430565345,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729915735752959,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833730279320949,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844833027179304,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865997502776352,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918657472161087,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925210243826242,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919997239240298,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928853651263604,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957996541389117,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963329936757586,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977394588977883,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978398415198746,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982146078469477,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993690516015417,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.33.self_attn": [
{
"accuracy": 0.9756516485327953,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9765013945043871,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793335626784124,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848526981169063,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881440394459978,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882685347999397,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929257548830815,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930772937858772,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936986656393856,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940930021019947,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941139511004287,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944649501581138,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948343906036922,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953536380351963,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970927490840519,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975688940306243,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979123313862243,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998686552749285,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994533700044389,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.33.mlp": [
{
"accuracy": 0.9644144453892582,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652636381552407,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9709799892612194,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9725714483739514,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829133491178876,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840255798281807,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863404538482428,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991631435965629,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922946960493726,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991754319494296,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926494669512307,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995663780780313,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962057370851797,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976682555984313,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977680962643047,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981967652994698,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993735759252511,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.34.self_attn": [
{
"accuracy": 0.9753232751727888,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.976149314034142,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784271912649274,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834901852729289,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876182950837048,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.987708578498936,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921317859185174,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922101932798365,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927493398396396,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932427918538451,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938270679773077,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941394149178737,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994371436781397,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949104583233987,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968183686601391,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972933884710073,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976280000256864,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985865674772005,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999356696737538,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.34.mlp": [
{
"accuracy": 0.9630964452302769,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9640003215325506,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9696978605107257,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9713053183237973,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822492907313924,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834059903300122,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857059930402198,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913165432302967,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992005208752265,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914406347088516,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923737216128135,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954862797902407,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960455132194942,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975459451149953,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976367631032573,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980421184874629,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991907467498861,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.35.self_attn": [
{
"accuracy": 0.9811779403764951,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817523460737184,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847730635420272,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886021953027108,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910593772993276,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912794741899952,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943893178194565,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994668104183419,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952095302900201,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956625883994428,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957341883993266,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996098927007743,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963174617947325,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967726542626655,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979197868899639,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982978617129112,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998487142582522,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990472405563771,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995200269094264,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.35.mlp": [
{
"accuracy": 0.9625688200130275,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9634864846930692,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9691977391607667,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9708253237859983,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819499335594868,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831378083480032,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854630313715652,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911853278447923,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918981589205367,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913226145956862,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922854665078615,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954456904731495,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960241412167976,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975546676616528,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976633212737445,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980702816116575,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993368028111658,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.36.self_attn": [
{
"accuracy": 0.9767783848862899,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774783570809584,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792888243437597,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833249478649936,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989179974029723,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892671639589887,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931318435303279,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932230088712746,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933254568666023,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939415423616179,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943516004315921,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946245992507198,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948382559354956,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952911886165997,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971430989313185,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975600748960125,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997801459878438,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988045292342768,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993882830986924,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.36.mlp": [
{
"accuracy": 0.9618892170684903,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.962828477648528,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9685850741929913,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9702174640015552,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816111180147058,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828095698337022,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851469866147167,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909914158235648,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917001000507489,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911248152340321,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920854259429401,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953006615641674,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958766914634524,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974178766500891,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975412998189169,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979500449385101,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991331418522781,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.37.self_attn": [
{
"accuracy": 0.9769636355830651,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9778583218882743,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801173926772255,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848725705064441,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884824765061861,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887249508480492,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927474050660079,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930272871309793,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934955647852468,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935724778272408,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943496980581825,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945753615619125,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994968130486086,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995292434897764,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971386417869086,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975643112802094,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978809258532956,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987350811440449,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994328011878104,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.37.mlp": [
{
"accuracy": 0.9607137066165083,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9616781197684376,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9675381602229256,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9692168731830622,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810064618818854,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.982253388109568,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846545533571196,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907162682349352,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914591607245568,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908585265572918,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918621958380467,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951881674902612,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957934863571274,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974119631990203,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997522525372915,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979606594240881,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993060297351085,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.38.self_attn": [
{
"accuracy": 0.9821101046706501,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830900786052409,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849303810072965,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.988099010342634,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917182464789795,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917944968972159,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949794864331029,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950817057764844,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952841070358103,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955791715704101,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960505152856441,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962563143650952,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965469543515754,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968721563498931,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980904909867891,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983816910596368,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998664444971119,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990784428113042,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996248594788516,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.38.mlp": [
{
"accuracy": 0.9598151959086719,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9608130584421911,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9667191931879834,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9684152805099362,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805380010879353,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.981828659674839,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842614569015017,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905008328098216,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912651660852134,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990649008192122,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916866350134736,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950846900526238,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995711345397132,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973607032964202,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974767103129508,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978991200669521,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992941810204158,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.39.self_attn": [
{
"accuracy": 0.977205573924278,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9780337426713422,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801262099491922,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846154492054331,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889356151321217,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889809511552908,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993236708062652,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932908966867743,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937063073494324,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940214214331814,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942271149913339,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947239582941524,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949025593722534,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953048436191717,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971289061451036,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975419033710894,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978729152843651,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987059635796437,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993625935151756,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.39.mlp": [
{
"accuracy": 0.9588926308053105,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9599204196741706,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9658451328348172,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9675586405945451,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800880859654985,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814001972738066,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838291065963475,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902547462843359,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910360393849643,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904249769292379,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914737941060019,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949559714203995,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995587959272885,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972762623495471,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973967088276128,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978323135108343,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992435063284478,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.40.self_attn": [
{
"accuracy": 0.9772713549043003,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9780010905018762,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810420040923514,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853588443396515,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.988955944727518,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888873245125931,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927413716887761,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926006464214113,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932274882632651,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940704860920576,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944877520718268,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946900564858592,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949137965511334,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955682796426117,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972269197521535,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976431035412181,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978965663253084,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987854673802902,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994290259921016,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.40.mlp": [
{
"accuracy": 0.9584853719723851,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9595296320162321,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9655107157795053,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9672348264996943,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.979896125530726,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812220203151044,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836761751947435,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901590041669184,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990949961871497,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903363397246913,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913953874309204,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949070829775577,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955472090453106,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972513176525306,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973765808145368,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978262450558281,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999260862347192,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.41.self_attn": [
{
"accuracy": 0.9895788403227925,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898260315439027,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991464405933297,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936754766018375,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949555819927666,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949855038062915,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970030344924644,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970356359109772,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972263467069226,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974329734441677,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974319304119011,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975670305215508,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977805044403986,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979844663512746,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987419291956987,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989480020533512,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990640054058627,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994652607144273,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997187499569631,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.41.mlp": [
{
"accuracy": 0.9757781559111256,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764562068801177,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800903448148778,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9810861255973577,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883986102466128,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891813759397912,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905991485146316,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943219732223568,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994772264954487,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944036066728202,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950298896843666,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970456683995986,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974259097239395,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983994044994592,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984698754882342,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987335165742001,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999554802832886,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.42.self_attn": [
{
"accuracy": 0.988456537211804,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.98876290069893,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903892953635046,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924727204549861,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942879665912571,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943775058062257,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963342958914214,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964246369996353,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966525912566698,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968158253468573,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971390152288797,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973043669444067,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974986482840521,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977304663533639,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985798271832449,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988340573618189,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998901112679098,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993965922530104,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996748602011634,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.42.mlp": [
{
"accuracy": 0.9746376424537677,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753612070589474,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792587450754485,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9802775859930798,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879440490254446,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887660526621499,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902118860841974,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941231506435495,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945796136621778,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941814388516114,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948429799099502,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969304635213982,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973292477845558,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983388413889579,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984081505834567,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986778153075935,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995383065091262,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.43.self_attn": [
{
"accuracy": 0.9869764895717564,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873284299514795,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890753223168615,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913325308282909,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937765555774891,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993815387035475,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961537277068648,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961651859032971,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966364346910268,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969684534191497,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969556399155408,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971532994195035,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973348599466446,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976307927128417,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984734453763323,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988255823263898,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99885263945601,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993481678171329,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996970330554943,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.43.mlp": [
{
"accuracy": 0.9729061192881904,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9736851985615335,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777875237755085,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788621002318043,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871344223716542,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880171223102432,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895180610596741,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937287515249887,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942215316757363,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937917997840008,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945041968511712,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996731321462185,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971579421238091,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982312779450838,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983115427618797,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985815367299916,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994927072165363,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.44.self_attn": [
{
"accuracy": 0.9860422041414207,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865772463626375,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879827184945738,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903859751250007,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927089827527341,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927981536663872,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948632083097962,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949893592121569,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995304119319802,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954329056077098,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961692341235712,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964338394592663,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965192379773056,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967695432163677,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979647877411671,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982858880110527,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982998594151515,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992023832517014,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995301218671222,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.44.mlp": [
{
"accuracy": 0.9720983174384424,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9728518091142178,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9770186443960196,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781182817624587,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986686219853398,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875984294340014,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891552355042413,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935172086238468,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940289788462809,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935745314180263,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943071327441814,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996615053497647,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970557060411298,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981739270733669,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982491138752697,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985349430230839,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999494523928182,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.45.self_attn": [
{
"accuracy": 0.9858187520876527,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862101714028731,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884242832317556,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905063377840346,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933680204159924,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932221381091758,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953932489856685,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951759893447161,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958366382205369,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960079202305918,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959795992012674,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967453849232314,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962537155155778,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971259691703477,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981297512361045,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984725671575257,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984241348023428,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992749554512557,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992897422757538,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.45.mlp": [
{
"accuracy": 0.9711829127351704,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719402938591022,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9761618708603477,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773000950287831,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861782799768997,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871126052580381,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887344575438061,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993259073357637,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937866432714815,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933290320424069,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940768315236231,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964881801095448,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969365944914323,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981060468187359,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981862930948601,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984849255752603,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994804198841417,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.46.self_attn": [
{
"accuracy": 0.9837535804039553,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841789998800347,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859302432129258,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885824170502785,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991667419255368,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917711436160301,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939540564205105,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940293210705644,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947527941365383,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949175979525439,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996060086769591,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965905571344161,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965073922158856,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970985038181472,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981209808488769,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985200898525747,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985625516274013,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990853397022491,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995480687686819,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.46.mlp": [
{
"accuracy": 0.970476624114733,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9712294090637251,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755289209026256,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9767106751372155,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857852892999194,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867536570435684,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884425271547547,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930728194916523,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936327796772515,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993150481206708,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939284980248072,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964016242455201,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968689714656457,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980674429281958,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981491552561669,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984457452481269,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994715228293851,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.47.self_attn": [
{
"accuracy": 0.9860993506582943,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986654782863824,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885740917617161,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991422783902013,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932861911549576,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933663590100447,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957761107943952,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958873482323006,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960506417188107,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996374164475128,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965455780957678,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966602936313537,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969320354713617,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970763933072847,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998191742759541,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984715360149398,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998551087812398,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992638024728207,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99960188978079,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.47.mlp": [
{
"accuracy": 0.9695646298166952,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9703204260256729,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747533228150324,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9759859486826157,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985326712240318,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863150581276339,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880819234653915,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928408182765308,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934114123400497,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929138302043277,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993709707939017,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962707597480499,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967494276536041,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979922139686287,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980742890988231,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983945690665247,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994444770762991,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.48.self_attn": [
{
"accuracy": 0.9854587407439555,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859821620691371,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876486272060949,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990535632829721,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929434283587494,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929971629018454,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955320943416538,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955964640983822,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957648471586014,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960599102226919,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996263327524311,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996623119547669,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967277275496408,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970249630829418,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998125468745313,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984834458875028,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985216156744986,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999231197904967,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995606643772733,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.48.mlp": [
{
"accuracy": 0.9689817698182244,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9697529035375306,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.974268508143723,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755386601839411,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850448601339993,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860501544010875,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878748330581737,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926868213193589,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932785494694192,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927823047619313,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935930357192104,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962039888272748,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996690488830944,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979539904375806,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980428250518775,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983760318718851,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994404298455553,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.49.self_attn": [
{
"accuracy": 0.9833008300905165,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838008567396748,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856259991463862,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888483344234134,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991866243267922,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919200114366648,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948229150184872,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948569066994088,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953089676870915,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995921038305632,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960756323569896,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963607274204198,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965327381046096,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969827356123316,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981504940752577,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984491511254179,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987035072776244,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991470584462053,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996131174074866,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.49.mlp": [
{
"accuracy": 0.9694139197664825,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9701530331451642,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.974687647123478,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9759756693322408,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852204003469333,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862185694326303,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880705588546238,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992747871474804,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933468306584186,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928622111849683,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936681149257837,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962481387253654,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967339441033178,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979776021117639,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980664392136723,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983993734441403,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994194173563492,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.50.self_attn": [
{
"accuracy": 0.9791099412954951,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799259280981986,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9824951557246479,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869594887683266,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898804096308979,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899604186219605,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938172098788384,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939138995189416,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994467624414124,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99479078654641,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950205438010591,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952796138577947,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956144077066136,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960410366153443,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997523850160896,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979065909712134,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981867043595565,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988977401547967,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994925560466456,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.50.mlp": [
{
"accuracy": 0.9694981348553771,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9702328015118837,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750433568597624,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.976363994475258,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985342911125994,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863034681122946,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988230073530423,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928100540572288,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933865306359765,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929119172922679,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993691661922065,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962713068939353,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967419359906527,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979919425983864,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980779638403634,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984365844621176,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994477995603933,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.51.self_attn": [
{
"accuracy": 0.9795648294844126,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9802640078491286,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823079467015831,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861869552221737,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899198202367284,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899667493361783,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935502542771006,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935988956498668,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937419781069222,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943786035169309,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946782613589772,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949421687664366,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951763393381905,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957575448421075,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973279091982955,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976828583626469,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979440729672971,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987890440020956,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994174924106215,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.51.mlp": [
{
"accuracy": 0.9684952444543964,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9692513413335148,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9740022018453792,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753373800041644,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848055966982716,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858010698502001,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877244643376846,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925583264741459,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931534270412827,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926618218642512,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993466041786106,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961279103851044,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966120843064824,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978951042591545,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979762429690087,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983187378292266,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993180275226845,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.52.self_attn": [
{
"accuracy": 0.983944723774728,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984509481705333,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869974170762458,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900320009328425,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926129912736973,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926108197725722,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954493738049152,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954022204527926,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959215498897002,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961095954119963,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996393136269609,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996638273973459,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968785010645852,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972413606990717,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982515372041809,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985531190299968,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987179371895955,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991969778815186,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995364703844596,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.52.mlp": [
{
"accuracy": 0.9680062588304281,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9687809720635414,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.973567819124774,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749357387619583,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845306066992251,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855594670056904,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875232116564324,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924319208795694,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993056461153748,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925462634449727,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933817586493924,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960871340927521,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965880982528784,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978983453720024,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979900134433257,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983375257702152,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994181337931781,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.53.self_attn": [
{
"accuracy": 0.9798742075401702,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804637919817316,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819443941998639,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851170033461562,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908354146111953,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908639640106183,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941107040442723,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941627457492838,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940711389124197,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948391868781886,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951494740640843,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952555365345784,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955533944854611,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957812318431312,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975649991967274,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978697604781605,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981082329606196,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989504801066543,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994558728925002,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.53.mlp": [
{
"accuracy": 0.9675464021336091,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.968327231018951,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9731538812384793,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9745244654875836,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843094536152325,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853387295611595,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873101021604318,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922972484736851,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992913007380833,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924095799460223,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932408865026542,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959854533601749,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964832969463283,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978015263086969,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978788904355544,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982238757473073,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992103628311796,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.54.self_attn": [
{
"accuracy": 0.9806514358834216,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813799448801499,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833217763685083,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872213992976436,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904578240147155,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906756263471356,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939281104720736,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942295323126018,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946022149616558,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948193393892756,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953221429727579,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954827656528276,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957975075016484,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960563984748564,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997587547576251,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979742295170007,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998168916205606,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989439094359497,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999479695920232,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.54.mlp": [
{
"accuracy": 0.9664991720530548,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.96731207676624,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9722410327705898,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9736531691037511,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837865627517826,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848543414846063,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868830000050366,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992056768534607,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927011576824283,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921796970608595,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930426115660291,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958857181849644,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964043466423295,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977843682342944,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978794785657603,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982517148355806,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993927469523011,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.55.self_attn": [
{
"accuracy": 0.9848535029021533,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856625254707116,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873553256651288,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899835190245587,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929435930557942,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930298070815441,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956733062005553,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957447823896808,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960009269171247,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960622051363125,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966529395406771,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967491822623599,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970109071950183,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973647626067855,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983738263131512,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986181395820735,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998847233165227,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999190504750004,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996643290030892,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.55.mlp": [
{
"accuracy": 0.9656929752152217,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9665306309532178,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9715057300697816,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729434285606993,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833657031290626,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844734465311232,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865369082575566,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991860639573516,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925290391241249,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919877524095538,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992884366577001,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957905209525243,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963300230079576,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977397536430018,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978369723110901,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981967572729398,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993824431669629,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.56.self_attn": [
{
"accuracy": 0.9810840039955158,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817251636597671,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835001551301071,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871968823534093,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907191593543088,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907713030513964,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943236012109801,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994346871518677,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947066721506417,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948807871949515,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952088647129896,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955679333989361,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957612884402471,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960301758623437,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975582133598724,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979204283720863,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998163968836276,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989021389199862,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994420239573197,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.56.mlp": [
{
"accuracy": 0.9649832576121155,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9658462989487147,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9708466551413661,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9722956360170716,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.983024675399065,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841407599221719,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862035019078145,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916705744782168,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923460454140839,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918224588036537,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927154623315131,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956916691735387,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962295474435546,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976659032369131,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997776012414282,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981486504578865,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993393813416142,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.57.self_attn": [
{
"accuracy": 0.9807673408894947,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813798840500807,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840419071383382,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875255209687901,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908318490917353,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990782226729942,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940294085781237,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938762680333304,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994269993474805,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948222570268339,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952707048628087,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954627062390117,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956676925311944,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961909005712521,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976283966343066,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980106372937658,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981702334572267,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989630067322747,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994832357379461,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.57.mlp": [
{
"accuracy": 0.9646938111829131,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9655546887140525,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9706002803225267,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9720574607092299,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828873196322667,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840176967217734,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860976392305211,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916036826754479,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992286872094203,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917594115739983,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992664294635975,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956584897406987,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962034562823215,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976532943240416,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977610003521764,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981456890175315,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993518922680156,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.58.self_attn": [
{
"accuracy": 0.9742431287212592,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749779423796817,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9782740444337067,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818783321564919,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880601780391053,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880831320279915,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919196551322544,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918741756118834,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992499563303825,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940836144194595,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994381465642762,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947813203509309,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995091238586036,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956519767813581,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972827841993421,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997733377514554,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981024301316785,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986767371822345,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999434344763071,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.58.mlp": [
{
"accuracy": 0.9639055783811369,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9648100247508601,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9700099239616018,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9714992932583156,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825402094835514,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836898146589336,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858239898948293,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914620976257873,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992153342641694,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916139009751772,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925397492602075,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.99559084485334,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961476971909992,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976242469243803,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977356102807742,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981065997670061,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993497312994135,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.59.self_attn": [
{
"accuracy": 0.9800955377715198,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812219718863305,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843853934128818,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882524164246493,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904496473701376,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990618176243611,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939009181444386,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941262882938119,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945284947630411,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947732121463081,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952919295572332,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956253039356517,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960523458124187,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963922690796224,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977501543103963,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980342253233845,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982105248521915,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989822886755543,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994935132700974,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.59.mlp": [
{
"accuracy": 0.9633515252682724,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9642617447595847,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9696107171475887,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9711391450347084,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822720791164198,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.983430200648543,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856252743323383,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913327464049584,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992012561960636,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914696740084573,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923932836636117,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955048800544127,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960630934464892,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975759088012733,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976821082926012,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980809420138892,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993404603799143,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.60.self_attn": [
{
"accuracy": 0.9785662742919827,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795471611187646,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822038880696422,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861939367593119,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892957268870974,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893998922289986,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928470086796504,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929293372888902,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934532373628923,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941524628696865,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994689533621759,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951109257404154,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953549808920606,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957487359642982,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972698958223,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977399925394964,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978288032089997,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988014610257538,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993713516098643,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.60.mlp": [
{
"accuracy": 0.9622851669984429,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9632317188539004,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9686702168301532,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9702322289935852,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817660457797741,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829566515001812,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851860736839866,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910901504519739,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917955345866319,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912368131703452,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921946783403033,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953845943397793,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959601003210992,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975101812191209,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976210160805893,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998025464917239,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993234407187341,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.61.self_attn": [
{
"accuracy": 0.9777745861854208,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786532930539626,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823275313859707,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863829188489992,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899176853757939,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900080686631171,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993900936415517,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939841359584151,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942143939092363,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945249848655964,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950129052280987,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954485604907141,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957426793162564,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961640784811032,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976880612023371,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997999354657766,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983797501527557,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988935410602655,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999532645384438,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.61.mlp": [
{
"accuracy": 0.9616354019626191,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9625926234416271,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.968127103424386,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.969732257096391,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814196676506024,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826445344737486,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849314519547319,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908925857100832,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916275912699731,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910640322456234,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920441526429433,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952813275035863,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958677367968974,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974189872400051,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975563913007806,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979488994975231,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991963266404223,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.62.self_attn": [
{
"accuracy": 0.9753544385005769,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9760525022192221,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792654263835988,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835837285867647,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878807245067468,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878389260633603,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919671136885881,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918816539605981,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922032521204337,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926502875689613,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935225938863465,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943081334502878,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941040656137231,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950546954488871,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996882195186213,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997322393771212,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975614156451469,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985117494637196,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992825674355692,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.62.mlp": [
{
"accuracy": 0.960998835042119,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9619702523280131,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9676230246887395,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9692554464073557,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811519819164747,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823793495368016,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.984717940078362,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907594566282473,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914899879919463,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909229083967052,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919129090726768,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952177098143453,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958124839406657,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974129036326256,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975317474025743,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979590275202339,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992785654107329,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.63.self_attn": [
{
"accuracy": 0.9690891312141168,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9700046626752928,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738487485109976,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793086691122306,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854851889080907,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856339021046695,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915185836879047,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917296289809441,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924033523390168,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929624833289141,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992959493452585,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934117309749126,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940434781172754,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945381800276473,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966793737288466,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971652476080252,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997763193635221,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984890966565887,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993746039506636,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.63.mlp": [
{
"accuracy": 0.9597413522240362,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9607453712899434,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9666322201098266,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9683507251504221,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805548783685816,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818357788516503,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842625613275328,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904371622557703,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912168332130501,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906309115161237,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991659320817378,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950620304106882,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995681538798013,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973245610875127,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974399392100933,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978871536992588,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992116852004792,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.64.self_attn": [
{
"accuracy": 0.9717008938130579,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.972536734550407,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9761840825116164,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813980772405079,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861987883127049,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863095470753155,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991650515162435,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917579466280969,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924410379883882,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924788754923564,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933275815932766,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938603797927499,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942497954164681,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994645845488106,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968310470729577,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973260171490869,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978329554494274,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985651758217596,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993528379602236,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.64.mlp": [
{
"accuracy": 0.958359407261014,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9593936365685964,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9655429655195851,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9673584922167816,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799075705048285,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812256381601879,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837793778432044,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900684410608128,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908760776743293,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990289738769398,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913495673650974,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948481073063847,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954770721838271,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971253918974023,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972762675698552,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977168430402679,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989181105598905,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.65.self_attn": [
{
"accuracy": 0.9687651162476916,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.969978374772166,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732413697791727,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794928112410402,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847885082524858,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985137569374944,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909531984870371,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914539622535047,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919543973611373,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924469225500759,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926874140571607,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931093100458384,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993614822282995,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940949650481343,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964249872381946,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969418459988543,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975369127320224,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983790771190175,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993234579007165,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.65.mlp": [
{
"accuracy": 0.9578814844747907,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9588999709016398,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652977152482459,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9671492813841293,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.979649099325271,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9809869766529453,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836303777012386,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899756592502328,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907894523471201,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901617287756189,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912353156526622,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994795575983038,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954403836868311,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971476831828783,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973186979646256,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978114063790264,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991771941728841,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.66.self_attn": [
{
"accuracy": 0.9700046873798496,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9710777752885693,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9744384925028211,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806345668376276,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853416310721322,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854729341245011,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913452750954189,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914812223103485,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919448898566005,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992658866128247,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929118402988503,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933832555911258,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993927660434948,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943780593794623,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965967290887707,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970890433882902,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976549701348535,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983906559707401,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993219098036398,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.66.mlp": [
{
"accuracy": 0.9576547232113386,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9586772554014859,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.965065296347204,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9669181533942097,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9795711703696534,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9809196138950554,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835560618850746,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899611657142247,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907784389675056,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901500096005437,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912326137878393,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948124769511387,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954659470822662,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971927947327984,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973192387750667,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978004249829897,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991916355762729,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.67.self_attn": [
{
"accuracy": 0.9696364225329537,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9707788810330001,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742607922832432,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801937266203918,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848824745828384,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853344095478717,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907987044977122,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914176264581712,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923394410625884,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929910828359425,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929395630269459,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934600547299182,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940237091290519,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945060635524753,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965786276458714,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971428266971519,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976746766264305,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984517151912625,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992667626310753,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.67.mlp": [
{
"accuracy": 0.9569374526801863,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9579622106332528,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.964379939299665,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9662555363029242,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9791792025789618,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.980595582213841,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832546634991702,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.98978246998434,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906432863668,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899831504413956,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911180444698977,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947448894871693,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954222415598404,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997163964178119,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973049348796179,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977676535184544,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992145522050314,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.68.self_attn": [
{
"accuracy": 0.9752179081306646,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9760448476789814,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787640092698368,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.984106579256293,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880659857666806,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882766963227799,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928722696525878,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931119007401561,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935542404112455,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938693700088678,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941538753306591,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944485194098792,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948950849939138,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952410595540545,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971185698553822,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975684361201466,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979447631202149,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986715006158001,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994181211498615,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.68.mlp": [
{
"accuracy": 0.9569247478717252,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9579326563367718,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9644662729023319,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.966374447765319,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9791938475774307,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.980571805695562,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832504067177835,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897252984442994,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905962617822776,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899607669366034,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910686748641494,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947167048966022,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995377654712157,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971233132589412,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972804920919436,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977774147471217,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991824911006628,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.69.self_attn": [
{
"accuracy": 0.9652934669467964,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9664122476977737,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9699095396423026,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.976600424288527,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.982828304563698,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832298012548372,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893747019385429,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.989943944603989,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907152076790992,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912593399027461,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916884254321063,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992135050584023,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926444555779821,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933005477007675,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956326659180617,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964500219732719,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996775866326827,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998185855973708,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990630614215901,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.69.mlp": [
{
"accuracy": 0.9557507448878727,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9567894279760749,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9635028269534048,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9654014880131734,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786359557979986,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800581724235886,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828113446894445,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895695198180252,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.99041610664541,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897113657360407,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.990851500561755,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945939456872446,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952770912603132,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970930032432079,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972142635746614,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977078650661401,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991971146082506,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.70.self_attn": [
{
"accuracy": 0.9774922213370079,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784128078210511,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.981250093122454,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852108885779193,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895225261877242,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896219971001541,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939037405787722,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939968753360996,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939534420539674,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944013164648297,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947557179747444,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950759799685329,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955296485047591,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959145393467655,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974643670843522,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979581994825582,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981783671169787,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988575970484433,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994799008020078,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.70.mlp": [
{
"accuracy": 0.9556012290873026,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9566327566771131,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9633668665038913,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652819453101409,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785209792715154,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799566167945924,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827256817860823,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895219351164997,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904008855013863,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896885575726628,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.990838638923474,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994595879688859,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952797767779741,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971025416468221,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972268176497892,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977052747945938,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991981418784953,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.71.self_attn": [
{
"accuracy": 0.982224799251478,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829506337348568,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847152155186785,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878374219528938,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915303558748412,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916317954561428,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945165154548656,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945928453173685,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946672272819438,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950314060000605,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955739959535238,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958986049444464,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961348445228252,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963391857515824,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977000909918723,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980546305887401,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998146002916129,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989940181043685,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994669277512003,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.71.mlp": [
{
"accuracy": 0.9554429884607855,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9564592018723488,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9630902379163002,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9649561430866781,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783923809759711,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.97984358511473,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825777198727194,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894790650394402,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903339552447984,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895943404959613,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907555401227192,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945343205656269,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952265932959946,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970687070591865,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971834441990053,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976686134713849,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991964394790366,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.72.self_attn": [
{
"accuracy": 0.9748019106490048,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758495617853967,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9791465982990829,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847738860959285,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880901199373368,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880637772087204,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933087012093318,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931614298471495,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937253565233397,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940912835299969,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941768885320542,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945425239857286,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950457909267003,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954108411789333,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971668615839199,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976084290966881,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980287572712099,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987052278103012,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993880458457983,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.72.mlp": [
{
"accuracy": 0.9556224923580885,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9566339712197843,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9632240603432843,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9650708399713039,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784523384449514,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799086517703376,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826108331448937,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894637980890509,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903328496668684,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895931850922736,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907555900710193,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945092442764067,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995198492414171,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970008041342034,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970377280842513,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974897910142317,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987637840983409,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.73.self_attn": [
{
"accuracy": 0.9838066731922721,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846019056674681,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863051298123441,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989320944994688,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923206969458414,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923254944089996,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995259349479487,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952448303075997,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952846075481686,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960157343333489,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961147991891363,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963795313685152,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996614316196524,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970830833343299,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981948575127477,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984685820098477,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987250456653879,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990932273856168,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999599064766172,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.73.mlp": [
{
"accuracy": 0.9560319448968297,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9570312198055417,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.963495558029727,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9653086666213839,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786056034070881,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800644926726818,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827467472733635,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895894487241381,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904441994516865,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896844435532234,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908487752913252,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945567106754568,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952420793767822,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970221518612418,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969847741365236,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997406517731418,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985942090616414,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.74.self_attn": [
{
"accuracy": 0.9860992159596399,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867376675829291,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881530592806245,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900461939457608,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929363591547468,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929878958746007,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960235994995424,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960586873471344,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964194025256132,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965419853351226,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966001983266324,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969268186802143,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971643206785972,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973495151651534,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983688430603299,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986626198791017,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987237121830205,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992952776067939,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999603938502811,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.74.mlp": [
{
"accuracy": 0.9561277217182674,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9571094210014531,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9634765605197141,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652619504026676,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786104874587372,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800704870078909,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827202689392787,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895987702582619,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904574185952937,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896853725720001,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908556570053885,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945776422973722,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952782527175977,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970955028639812,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972001499569926,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976683519106653,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991916658258752,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.75.self_attn": [
{
"accuracy": 0.9871757453013408,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876340927164021,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889538600354603,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911560471610803,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935592690139616,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936098377220333,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956273871449459,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995681365866116,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995645305362383,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961984071905088,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965844685947033,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967272645965415,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968482684956765,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971318664029241,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981351793363789,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99841284862793,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984292147288981,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992625033789265,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995416154974679,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.75.mlp": [
{
"accuracy": 0.9563265087965288,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9573138982645775,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.963573111907432,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9653217221953367,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786901150486971,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801594009132761,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827756497420763,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.989641643423391,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905101609612373,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897335873622644,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909009621968787,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946038109299383,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953007570492398,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971105477306992,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997217978840988,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976752391268843,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992078798210347,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.76.self_attn": [
{
"accuracy": 0.9913937099964211,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917523652854326,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931446165826759,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949635561482099,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959069169616621,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959505855497953,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975752603495494,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976029524359068,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976778287489555,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979118302366451,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979677588041675,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980746352015749,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982914362151764,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984209772665054,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990474079355696,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991607931044296,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992978487142338,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995166099703822,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997371871424164,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.76.mlp": [
{
"accuracy": 0.9768827157585245,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774377899930665,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9807703406795075,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.981721032361843,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888417644328192,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895883682919177,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909550370893588,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945143277903921,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949574356310462,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945945849544123,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951972360845263,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971498596040826,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975119305492722,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984512382713882,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.99851779995094,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998770189303047,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995349118544254,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.77.self_attn": [
{
"accuracy": 0.9902164336097868,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904870930814037,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918071123967437,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936595764011145,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952000905514548,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952494046087131,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970295670760894,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970774554622996,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996999137422168,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973939016069236,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975356164964309,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977277140561981,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978653905533352,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980532310927581,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987593168227346,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989546121857864,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990447198690259,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994179536251498,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996505129322606,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.77.mlp": [
{
"accuracy": 0.9762510376933374,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9768384735247022,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9802819705127102,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812668919759361,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885554228860297,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.989336656970217,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907319030087245,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943762202502081,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948439992425081,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944609486939091,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950930075842495,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997081114163034,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997458723214406,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984153887734896,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984835692142185,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987400836237755,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995253162525938,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.78.self_attn": [
{
"accuracy": 0.990130222353496,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904427475443012,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920963471461284,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938591189780518,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952917105493773,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953418361562255,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971088665218926,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971554067696592,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997277646999512,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974717889716359,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997708370069679,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978558125194946,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998039009815425,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981951678208517,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988947465102874,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989958381231286,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991963732693541,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99936955700028,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997149732761931,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.78.mlp": [
{
"accuracy": 0.9757445655567082,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763447395280788,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798530271570933,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9808616112721594,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883334793052391,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891212298289725,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905426690943147,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942427759401893,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947217536330419,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943446685609064,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949836280736092,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970108693171489,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973909443697745,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983506895675275,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984176184822756,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998666416988463,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999390612423763,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.79.self_attn": [
{
"accuracy": 0.9888101754906146,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891511765798848,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908772649880695,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925482493188036,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944727443354694,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944847531763739,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962838331580555,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962765131586868,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996447166881377,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968771573989407,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997325877671277,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974490977750209,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976922786659806,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977956917014366,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998648236640484,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988677777092609,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989227834464002,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993835177768855,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996286437105338,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.79.mlp": [
{
"accuracy": 0.9752468616750679,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.975856234466559,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794596957631017,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804921966340197,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881118136507115,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889090914083155,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903697981744221,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941339659455576,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946140165832874,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942406301974858,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948822999569146,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969638712017944,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997350556996504,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983442002159887,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984217467840368,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986947916515515,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994950761902146,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.80.self_attn": [
{
"accuracy": 0.98618321071722,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865693577908372,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885303024692755,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905821210459659,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933038056877098,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932906854113466,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959105469752103,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959961828457093,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962572734680419,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965777938180652,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967217766924908,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969270910430503,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972788192344045,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975405031337256,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984858146942172,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987256011717316,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989092926200676,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992856953820017,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996393915743714,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.80.mlp": [
{
"accuracy": 0.9743500907758349,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749949919550043,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787229146122148,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798067082209807,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876950490043351,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988531393734248,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900409290812126,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939082901630747,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944155014757263,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940284930512702,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947053097237489,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968522749473586,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972573032872262,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998281295827933,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983497603281745,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986339975025897,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994307657133935,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.81.self_attn": [
{
"accuracy": 0.9866969509932556,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870675628710734,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890170143917203,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912553911440467,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935468749486303,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935932828260488,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960226318921501,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960707426806422,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965490968285227,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966424289649647,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969349253224209,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970995378496713,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973203815997725,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975489417801758,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985107309123697,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987339212675579,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989559658386401,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992822377213375,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996544946119875,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.81.mlp": [
{
"accuracy": 0.9731638696240751,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738221208711988,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777518811782724,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9789080292378601,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987101474727847,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879828038576403,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895917309791242,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935833961016646,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941273095380319,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937268350959608,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944342289512095,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966742888975301,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970906662362579,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981344548899582,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981445949340827,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998417354366546,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990899424914172,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.82.self_attn": [
{
"accuracy": 0.9848808154561802,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853568307957367,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.987377380094442,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898958938980573,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925464476507745,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925891796715165,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951931561931575,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952102019329017,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956975177567648,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958811999225107,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963411786716039,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965657512902429,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967611034180185,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970746505576628,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998187449509523,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984184260124733,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986360230740454,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991424176351804,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995838661025597,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.82.mlp": [
{
"accuracy": 0.972630397955838,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732699140318131,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774021977852834,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785916597435349,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868079364103707,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876925222281563,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893730117841378,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934555873225787,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940072755623413,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935848604348537,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942975580496224,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966017955693564,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970290404376819,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981220999646834,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982341290696671,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985545886263839,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993996884379732,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.83.self_attn": [
{
"accuracy": 0.9848161959334424,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852466827356502,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870739593905838,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899885678467782,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926285942722308,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926874138856012,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955443910960304,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956171198895103,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959995734971017,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961280832924929,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963733126948539,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966295704637703,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968957687225682,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970715432305282,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982203701878652,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984801957371497,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998727711288283,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991323962887307,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995702895026434,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.83.mlp": [
{
"accuracy": 0.9721790473712119,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9728408801908556,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.977039965751924,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9782449248197832,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865992518730069,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875140746467208,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892251412139127,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933844782215985,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939492517396024,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934966812449458,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942374635536811,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965764528115917,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970203498378396,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981411674831945,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982228660527127,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985431953345573,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994378657536703,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.84.self_attn": [
{
"accuracy": 0.9838998258407963,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845477033798632,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867664294709501,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896063513465618,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920930642095444,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919263384991178,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950532481245893,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947633092095586,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957774683318444,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961518408289474,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962017054203898,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996421186326954,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967478529770711,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969945128418898,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982018296386262,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984690638988181,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987302678729113,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991426930768954,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995501125721555,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.84.mlp": [
{
"accuracy": 0.9714876908416811,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9721507474191879,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764205382058495,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776532946056441,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862258546054363,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871754662967042,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889334741685736,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932160328730548,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938042181985158,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933319245220015,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941027355135271,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965007235050986,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969610334128925,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981097918383679,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981946859104363,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985084608381026,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994439708335496,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.85.self_attn": [
{
"accuracy": 0.9861853185943082,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866203592698041,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885096584054592,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910118580844841,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933927597379998,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934358302699892,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958781783444512,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959150857512692,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996230167692135,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964617779233346,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966458537566819,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969080670761239,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971081260101575,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973784602480009,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983877742810077,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986356596560463,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987769691383859,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992458122494434,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996093048723292,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.85.mlp": [
{
"accuracy": 0.9712471620816934,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719034533359503,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9762489903522166,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774959758904419,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860898880288005,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870447356155828,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888152410031149,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931166735722831,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993719963577429,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932514392329674,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940145748275283,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964404812352242,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.99689810185105,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980516851708097,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981594339846388,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984938056931194,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994150289031064,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.86.self_attn": [
{
"accuracy": 0.9816320872816601,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.982181038266342,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842819903339994,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874563934281468,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907606412215453,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991028129711355,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938314095580656,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942067253991569,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947981417399684,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950265014225519,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953793569247386,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995616995178065,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958725401175845,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961992835282887,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997589554815357,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979402537911052,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981018542863527,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989839141459302,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993757714600131,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.86.mlp": [
{
"accuracy": 0.970513543408168,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.971176603907033,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9756185139872526,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9768677258766011,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857731787487864,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867258018470908,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885408301886759,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930285277874454,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936055177087454,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931243180395349,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938929693185186,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963881524377748,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996845972025767,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980508649635962,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998131738871483,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984598864041465,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994320553439473,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.87.self_attn": [
{
"accuracy": 0.9871068159608465,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874985383235311,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890731328265056,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991231467182699,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939383661595026,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940024698030596,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963245354470258,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963732099185061,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964798955295823,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970212268863657,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969152738217657,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971373521931175,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973402177351282,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977041350518304,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983993378026705,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988093693912226,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987536401757481,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993151860774225,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996281374539984,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.87.mlp": [
{
"accuracy": 0.970328069144958,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9709875510123215,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9754543560311982,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9767118568875288,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985633097589016,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.986619295356305,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884440067077154,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929714781036111,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935774898009473,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930706004072961,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938633927683297,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963678786727158,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996839266968891,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998047017345303,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981272317481374,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984436839095954,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994293677185564,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.88.self_attn": [
{
"accuracy": 0.9899445126617425,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903448500523442,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915143822221771,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932051692403069,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951317273570519,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951866977529502,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967651233003524,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968360100725764,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968864364230907,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972432894068525,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973275149118548,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975168649168489,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976200850582436,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997817504875313,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985867509937012,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988078386430923,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988146983393419,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993836286119308,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996167385160286,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.88.mlp": [
{
"accuracy": 0.9701474968736109,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9708208239784366,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9752067391221461,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764351282072695,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855472147954922,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865223715866083,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883262219985849,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929410989494308,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935215845409977,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930208030058757,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938030392666789,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963350507406223,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967994201854852,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980263871104015,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981052915406364,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984264349699706,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994270909978935,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.89.self_attn": [
{
"accuracy": 0.9840167351067066,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845951913218749,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.986769191651164,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903741788217112,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923254922277441,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924342639401162,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954391764476895,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955351469964769,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959200120555531,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996232588579388,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963081288641613,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965177746612186,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967430625601035,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970971899180624,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982222350816684,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984687438148907,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987434411550088,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991451479941852,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995700522206454,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.89.mlp": [
{
"accuracy": 0.970087479505884,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9707471507748491,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751254217209,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763530687870163,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854840258822629,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864819577748054,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882738874048779,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928919720022302,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934890832702973,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929733405281839,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937696863505009,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962975601175506,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967680840605968,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979836621603585,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979947046254223,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982950047246719,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991491501783266,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.90.self_attn": [
{
"accuracy": 0.9900142677422417,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906412427999863,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916696813888848,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934507515281439,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953374946857557,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953606352332587,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970572589357433,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970794417160121,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971339933476165,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976408586923131,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976967837306132,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978387053185878,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980085710627272,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981893143052921,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988797745410058,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999058148471369,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999189119249901,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994096495931061,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997014803811908,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.90.mlp": [
{
"accuracy": 0.9700962584465742,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9707554910135897,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750917003931183,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763050657746039,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854688100320729,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864581561598339,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882542329203141,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929151881829297,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935061958079275,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929836460291163,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937785961175043,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963065120391548,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967757011763752,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979907809695425,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979886934467542,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982788694140158,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991279325239655,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.91.self_attn": [
{
"accuracy": 0.9908690828290817,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913114865898693,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921110664298268,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932341010456807,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961648164684639,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961920713662709,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974349304102361,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974884521216154,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997644364588747,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978143981976533,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997982374496611,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979938040548739,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981466092806506,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982918982001904,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989605735815865,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991047360572817,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991409709235948,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995124534792634,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996975726389792,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.91.mlp": [
{
"accuracy": 0.9699400142815552,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9705983431715715,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749105760925695,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9761168444039006,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853530561453417,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863603027341398,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881489190125936,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928576753376738,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934534488834048,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929208399629906,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937268361988428,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996276025157912,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967580446115646,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979974097253657,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980710882227868,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983862243457943,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994130715831967,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.92.self_attn": [
{
"accuracy": 0.9921702239662409,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924359198071455,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993280114262904,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946302014991248,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960295152482822,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996054725243563,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972800870646575,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972843451204857,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972347833926937,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975143359287789,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978283647950249,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978461344971469,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980115203775073,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980980694771892,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987765336623661,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989669878942598,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989394012158491,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999503389961319,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996474984815761,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.92.mlp": [
{
"accuracy": 0.9698566878704649,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9705162442436344,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747750100336576,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9759627927682901,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853137746747387,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.986324402895805,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988100484993897,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928445933680785,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934452750573033,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929054639988432,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937158410290354,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962706122008201,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967517878639659,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979994447142082,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980707851079244,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983811170095578,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994218660990315,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.93.self_attn": [
{
"accuracy": 0.9909207836577767,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912899544119442,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926076137558802,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940164289927405,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959319767689234,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959571687259564,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974899236725545,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975168319713128,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977486928753359,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977444088221282,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979600703993224,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998130276198744,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982736914014247,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984077994564646,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990239008145995,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991659757736335,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992957044496985,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994898341767686,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997369040624777,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.93.mlp": [
{
"accuracy": 0.969658556443296,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9703194156877304,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9746171918354536,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758145005295151,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852083544982108,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862126563430617,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880039764960346,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928337097364036,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934233012854269,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928885775765306,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936917453367067,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962671975066003,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967458281102345,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979999264591912,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980711943629247,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983690217027048,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994067224952711,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.94.self_attn": [
{
"accuracy": 0.9868101933971047,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871470693400816,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883680785761068,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903088105891488,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936054335465949,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936913164381526,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957401990571892,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958455629377184,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961510054816148,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967364763693982,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969249914008144,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970275919982478,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971958963126925,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974484671838582,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983913114441461,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986238650744781,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987537725215876,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992540405609792,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995605288483937,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.94.mlp": [
{
"accuracy": 0.969439549665702,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9701024583099704,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9743842909994879,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755738028570226,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850965727512774,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861077263853267,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878821462686909,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927687408696664,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933551847444553,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928167661602952,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936188624093407,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962177142917522,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966969645802716,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979655639648339,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997990583045114,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982918803151207,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992519421533583,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.95.self_attn": [
{
"accuracy": 0.9924151054828575,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925718047214966,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993270003923068,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944359198408691,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965064243388999,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964730381524485,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978026613151949,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977384807762543,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977493592182567,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979741183281141,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980000572793774,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981649416854212,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998214664299188,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983375919860249,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989491917811776,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991018436787846,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991182910315203,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995515601767693,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996887372399215,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.95.mlp": [
{
"accuracy": 0.9693236418656612,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9699960928410292,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742477534240798,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9754238491387743,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850331297830531,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860413112236481,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878118490114024,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927493880355829,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933306508158383,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992786758995958,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935912708751857,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962071775777364,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966882180765664,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979690177255849,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980355410983688,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983415200901953,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994130289460851,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.96.self_attn": [
{
"accuracy": 0.99196294758861,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922962067648768,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993299284361695,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947124101148036,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996248813382791,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996250412880296,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977787871489694,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997751146610091,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978025761665776,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978772238460615,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981388520498417,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982383903789971,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983808183797488,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985636328101942,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991181043105347,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991899365988118,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993806060336187,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995346253302781,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997563052556392,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.96.mlp": [
{
"accuracy": 0.9691411749900956,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698149043282396,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.974098142726641,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9752897785878495,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849431444645712,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859570984875685,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877502151617878,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927201627165472,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933109737099394,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927657275509677,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935778258123288,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962037064772296,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966876060383296,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979732470329556,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980427791658593,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983394918534415,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994207308183448,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.97.self_attn": [
{
"accuracy": 0.9869195474066624,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875828500493968,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892156030982733,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919699739272657,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993888180261772,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939269801895869,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996447264504815,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964741564196485,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996676537655539,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967675958024828,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969112039053518,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970800396005967,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973450236598422,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975116269748756,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984499460980786,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986736139766627,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988068722733786,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992893187797898,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996292488986479,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.97.mlp": [
{
"accuracy": 0.9688880245544409,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9695704439164776,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738473866723085,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750332235309639,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848013917768472,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.98583436580865,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876222184025928,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926363182695288,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993238047843701,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926791242813986,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935038567177559,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961500273355725,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966411819485458,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979373530215142,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998007189304764,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983168603349967,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994037576957213,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.98.self_attn": [
{
"accuracy": 0.9897131091719004,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901392629958297,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919428213401452,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937733892400405,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953550578977325,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953991934215944,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996841929080945,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969152838579918,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970743225176671,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971785410674975,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975788795793625,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976943464533082,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997810733688407,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980175817189249,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998701105847048,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989080003456605,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989077648527822,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993356474518996,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999619954057332,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.98.mlp": [
{
"accuracy": 0.968793132783551,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9694773594995862,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737607228912806,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749511639146429,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847675323682396,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857851664878821,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875714808310333,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926217366500121,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932189755337802,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926757913720059,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934857094317282,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961496921084625,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966320700355267,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979384684212211,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980050553751522,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983161742504882,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993977117349124,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.99.self_attn": [
{
"accuracy": 0.9913999666459858,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917496974138837,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927210981075308,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942125106279395,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960445276412525,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960750793964651,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976832592112356,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977186183739257,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977599122181633,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979566937667856,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980995365124392,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982051953403769,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983662757103479,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998503033102392,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990632970553921,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999199749641488,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993265170885838,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995018953861865,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99974541244094,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.99.mlp": [
{
"accuracy": 0.9685408557324033,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9692388298480135,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735025719396377,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9746882702763143,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846231059024209,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.985669793561101,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874582550929565,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925677052110826,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931712582413303,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926133865343505,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934437379584109,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961187534622455,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966162965804535,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979236686945354,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979926278067165,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982883266148794,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993878988852406,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.100.self_attn": [
{
"accuracy": 0.9911316281282588,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917012910407624,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928845196080051,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945442942216208,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955701445793047,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956007989270514,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973994219602135,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974536932882314,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976026562385654,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977324537773568,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977416810457056,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978497676355275,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980767259104667,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982479177844269,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989079224147932,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990843029135201,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991605214002591,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994960993430332,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996965882356131,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.100.mlp": [
{
"accuracy": 0.9683671667191543,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690733017694009,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9733301913855892,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9745123495200747,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845443415994707,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855918885747853,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873733566093602,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925211708453533,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931232245442899,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925649092091542,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933969183873973,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960861444277199,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965834436374471,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979021782484395,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979741076117774,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982829510393602,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993985993726375,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.101.self_attn": [
{
"accuracy": 0.9918264743724936,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922078832689869,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931829695001637,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946158171484345,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99615938335314,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961359179578722,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976419355708027,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976088692323843,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976763539077518,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977094412254366,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979449458309988,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99804900693217,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99816801857914,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982692158756483,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989077137464559,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990611081348585,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991113833690944,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995021049261069,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996901848062407,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.101.mlp": [
{
"accuracy": 0.9681455435133294,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9688481837511063,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9730939184756655,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742684575582021,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844400725376449,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854785812234408,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872452130688256,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924422073374061,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930450941779112,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924952524812206,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933182963807332,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960288364856847,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.99651558002408,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978304114866707,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977747422986125,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980529282045992,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989252345985733,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.102.self_attn": [
{
"accuracy": 0.9928662599505562,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933006828650832,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950402365678823,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996167656426367,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965322497913516,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966412042880333,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981154214382466,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981549304456597,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983153981646817,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983660103508124,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983465183759108,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984308241952309,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988206526828244,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989122277444327,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993346106570434,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994124001455739,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994602852216063,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996464125847284,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997812701070583,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.102.mlp": [
{
"accuracy": 0.9678324480590067,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.968545611653673,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9728143854360831,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9740058474340721,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842780795634577,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853431347659544,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871357548981905,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924068146532303,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930214708867041,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924598030598932,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933038187997514,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960356768294188,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965421221047444,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978799960196116,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979497484960839,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982455484997971,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993777643754392,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.103.self_attn": [
{
"accuracy": 0.9936552059738651,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993994428996781,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949280543277335,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961828116425558,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969932967245481,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970105761404786,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998148667681227,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998159352518422,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982630951072726,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983726247415976,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984137917942318,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985012752034603,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986275351134521,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987147531907172,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.999152664349439,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992637627047339,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992910937626699,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996110858100062,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997383655450234,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.103.mlp": [
{
"accuracy": 0.9675612788843482,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.968291248930128,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9725793497146744,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737704828577606,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841581574200016,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852120402318082,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870040471616545,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923369038212848,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929502330181238,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923914707461862,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932288314019772,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959956457544314,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964955755705504,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978564604259047,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979274961607236,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982385492185131,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999391378162392,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.104.self_attn": [
{
"accuracy": 0.9892599827266837,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896405155427361,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926028803824202,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937049708770294,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946775577325178,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947963545687104,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969875941690254,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971417082767737,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99746155991268,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975352899391988,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973950726470273,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976066561464808,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981565337848702,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983352619333585,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989901963823573,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991266124416143,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992100451889104,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995633402186764,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999707599631572,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.104.mlp": [
{
"accuracy": 0.96718892925664,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9679145805145565,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9722123707791692,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9734028209687063,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.983946611075417,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850342747706332,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868225196474477,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922099516874081,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928343850678127,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922683795570935,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931209102812174,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959049918076122,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964110373739937,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977605925921939,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978341986241407,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998132996120196,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991854460033784,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.105.self_attn": [
{
"accuracy": 0.9930086974054575,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932284676960033,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994337988196333,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954972765040829,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963541120339773,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962957859725544,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978428113788954,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977843667392766,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998115332411161,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982489495856786,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982974093505427,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984710129762167,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985972418002539,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986834686715156,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992098187672367,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993387375143357,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994118671548724,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995949658853198,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997422997954633,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.105.mlp": [
{
"accuracy": 0.96681407025378,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.967554875404427,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718659999535272,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9730693385201065,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837680719792843,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848647652880141,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866711223301919,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921536262294179,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927912923556409,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922240167169979,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930843342046597,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959081430355773,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964226136768335,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978039977201996,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978877266954729,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981851861383276,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993646138418147,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.106.self_attn": [
{
"accuracy": 0.9900536714121699,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906960836317587,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923495498338812,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942117865736547,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953487610910088,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995401180079697,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971737142186612,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997216972442785,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974253156442979,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975277170122258,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976918916524339,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977972737550246,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980654930573349,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981766532935006,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988581242474825,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990224956724744,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990737141710461,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994439762085676,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996874880246316,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.106.mlp": [
{
"accuracy": 0.9664946033766395,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9672505669295788,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9715522529655382,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.972751936238063,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836235988493028,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847194957301805,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.986516644884097,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920737749014638,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927093137888924,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921360496725691,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930043737158963,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958562722160039,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963759273217109,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977757137048205,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978535194317565,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981638192824137,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993626224214064,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.107.self_attn": [
{
"accuracy": 0.9946575640749774,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948827024843347,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955917434080651,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968493873204448,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973079906837818,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973738793112141,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985225340259556,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985565885282611,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986225789700831,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998674291679268,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986689056416875,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987352637414771,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988325567162399,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988969335957479,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992830144596825,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993761174494131,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994213807666185,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996640085653206,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997705115146298,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.107.mlp": [
{
"accuracy": 0.9660728320871529,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668445008757868,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9711892705802855,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9724045887001251,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834076757297704,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845228844175213,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863399684331134,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919740549092623,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926155188709105,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920329092266528,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929166206060663,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958021035908085,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963313755824378,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977498344868716,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978284301101475,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981409305718875,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993650798201806,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.108.self_attn": [
{
"accuracy": 0.9888936545895902,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894070896369062,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922800990507791,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940100775075782,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948495665663167,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949389724796148,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970816335591831,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971900864835143,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971907585798028,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974118666968456,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974926789107389,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976034285561917,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99813521459184,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982808282978735,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989693284426865,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991036038570057,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992032055191598,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994814485406479,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997108277526231,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.108.mlp": [
{
"accuracy": 0.9654780284158493,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9662654504768158,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9706517707948622,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718827614070553,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9830953033739015,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842565096424598,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860904810852126,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918221847987488,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924912331625819,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991893452788262,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928076301799401,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995727524921102,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962747186226281,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977000926400682,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977837909307135,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980848466940714,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993038871708497,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.109.self_attn": [
{
"accuracy": 0.9935459556229609,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937887397888852,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952865305944885,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962915477686023,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968179533883047,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968542181641648,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977605576921058,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978176815447545,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980612663294826,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981940896985563,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983574955074704,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985055250581354,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986130800098181,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987424248798848,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991744695243573,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993229541869368,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992696827976033,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999608259289648,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997092339525759,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.109.mlp": [
{
"accuracy": 0.9645803627607069,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9653754072557938,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698321638922942,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9710848771623874,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826721029944325,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.983834741048907,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856949879444743,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916078415699303,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922772784913448,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916856089900983,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926003968882325,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956152143422514,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961632541380823,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976431540409593,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997733372294246,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980547887297642,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993316429945959,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.110.self_attn": [
{
"accuracy": 0.9883736785697309,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.988821452189433,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902241248637438,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922248120787308,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994452274698568,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945150435117907,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967288666122937,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967627803194582,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969231069431101,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970968887761333,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971514858587301,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973102284926235,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974837859994486,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976732545520032,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985415833898036,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987585227936506,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988792444897914,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993602279650268,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996326803382928,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.110.mlp": [
{
"accuracy": 0.9638646790071538,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9646811471565774,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9691445910812992,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9704074692961416,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823191890885171,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835192156759532,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.985381373106257,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914239820485052,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921191046504598,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915112074415543,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924566948688344,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955206302386758,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960887083955305,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997591412479156,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976812725592601,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980039103998264,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993166014794138,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.111.self_attn": [
{
"accuracy": 0.985318343302137,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859615229070187,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880207142743626,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904173892481547,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929692751580947,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929707079120961,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956794984785742,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956576118466297,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961199075914919,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964076774930092,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967186854551792,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996780526995855,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971219707819584,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974175266627419,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984385470559477,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986747974578879,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988805926619343,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992592658918032,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996493959211206,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.111.mlp": [
{
"accuracy": 0.9630784767826921,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9639325542865615,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9684305537099901,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9697084545500969,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819401630053395,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831486642850857,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850283096495428,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912235013111249,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919185704110485,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913198789091486,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922715472547632,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953966115380785,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959603625619294,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974635385565067,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976448794018085,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979536991419369,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992797758344463,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.112.self_attn": [
{
"accuracy": 0.9885926644170755,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.989071854439221,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990719626265529,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928601771029398,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946290684839416,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946385125727638,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967596111696606,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967520123372149,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970677057464951,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971927857320559,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973857389812014,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997575313931233,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977996610350123,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980080178509908,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987435020831119,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989540404732045,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990795984386319,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994020436363491,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996749202821918,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.112.mlp": [
{
"accuracy": 0.9620602035796956,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9629454125503176,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9675076896030652,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9688185587721435,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814248995757416,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827023273529975,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846146280730241,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909867524218402,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917191792720634,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911047895193884,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921022098217356,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995315940768801,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995913121273349,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997476828002714,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975813053315505,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979011438737967,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992787799335354,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.113.self_attn": [
{
"accuracy": 0.9871227975542608,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875902847730016,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898432129994035,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917840335861241,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937881068571618,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99385870463754,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959663333567349,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960443020528672,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965905523246252,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967999184798253,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970242914216789,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972096236153064,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973997789079716,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977897575161861,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986322808973981,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988515952902258,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990021855184367,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993484276867668,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996527989064098,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.113.mlp": [
{
"accuracy": 0.9613805572061163,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9622933240117211,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668923767381593,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9682263872145038,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811211671974314,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823894185063086,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843105765451726,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908261724110496,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915658999724608,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909586080251948,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919575949591634,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952373435769818,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958372074546978,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974373717682044,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975436074769517,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978634675499052,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992746094005844,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.114.self_attn": [
{
"accuracy": 0.9853342738688776,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860054596670365,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881564415991306,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906494765003261,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931206883691055,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993195929932163,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960709739549968,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996132596506198,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963342636137417,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965683878644517,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996690598831169,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996831819439601,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997162097482942,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974090202625743,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984554470437089,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986549563289276,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989211799815836,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992240295135784,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996425090081941,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.114.mlp": [
{
"accuracy": 0.9601210024403899,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9610689754940962,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9657657100377899,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9671410619232216,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804957227193212,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818269964797717,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837917237493553,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905083256675616,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912838494209083,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906564602187198,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916998990076152,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995080353643157,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957065275674196,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973492358970505,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974631777792973,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997791044291501,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992556094044918,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.115.self_attn": [
{
"accuracy": 0.9865619031418311,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987036659896962,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886770018033291,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910954229141536,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935932423380253,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936395947527337,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962449849117547,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962779322737142,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963813774091633,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968633833525115,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968850477659879,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971161433189225,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973191612547165,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975318237403897,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984969914719266,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987325538225521,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99898334339837,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992894080825959,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996390317499294,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.115.mlp": [
{
"accuracy": 0.9580374569877198,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9590578712522984,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9638801783715424,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9653038986419377,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794738596599353,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9808882810665589,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828985011891315,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899695888161659,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990791075152198,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901406278490628,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991251582157259,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947961158927923,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954620661998266,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971689382984646,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973123659882205,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976492683697296,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991784440139063,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.116.self_attn": [
{
"accuracy": 0.9818959768469396,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826323204232674,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854787635665975,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883479974967869,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991313825631024,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913300212815797,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950191637893256,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950132610791019,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953842359153848,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957458821836075,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958036687396663,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996080912122699,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965425421924967,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968131375437799,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980997168826625,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983799266751463,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99868413331416,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991187073878551,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995821994854946,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.116.mlp": [
{
"accuracy": 0.9573081601411104,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9583355501098068,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9632703439381561,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9647435455730087,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9791053816872207,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805367176764106,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825960823677873,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.989756642203582,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906001814739093,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899610260158385,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910876288225776,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947034546272143,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953866793627018,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971324489819572,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972700073726868,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976166873545337,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991927780762413,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.117.self_attn": [
{
"accuracy": 0.9808707098035436,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820292459702805,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.984154708291355,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876910331904104,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910792148868112,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911245064025647,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949714631013769,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950016976432189,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951865425897076,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956223751467309,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956549728828433,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959179531341713,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962680724473965,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965840837746662,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979183397770516,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982100284633864,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986152931508657,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998993591316608,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995662967925646,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.117.mlp": [
{
"accuracy": 0.9552078848998797,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9563083699659297,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9614170574043926,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9629578243352865,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.978071295166094,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796214580143753,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817131893023064,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891069525558698,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900374169412413,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893758057028448,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905879964414788,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943285590821975,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950489183814314,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967998428711373,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.996816376162889,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971559477577868,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984206294420322,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.118.self_attn": [
{
"accuracy": 0.9688286932283326,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9699714819067403,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9740690102702693,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796160906553268,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850285780665121,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851338164880872,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991417835932225,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914749699252609,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921035661215061,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992894833858468,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927805487654711,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932730989530683,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939762142633921,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945358141864601,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967072006609095,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997176833940964,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977824300784912,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998416712280902,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993471047721534,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.118.mlp": [
{
"accuracy": 0.9535078857290118,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9546525688155701,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.959922058703868,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9615495039248153,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771791975945234,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9789037590258216,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811040835553094,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885770688813768,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895734394851484,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889679229818285,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.990205792906253,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994095215279805,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948268023956763,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966185718015033,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967971031652078,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971757915214097,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985413856890151,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.119.self_attn": [
{
"accuracy": 0.9836657053643936,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844029732048512,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.986314347258916,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890194464180815,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920018297178965,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921240493851272,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954296114094752,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955809835810214,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958542137602834,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962124178725246,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963159865307573,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996539626393075,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968791334326134,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970864039282069,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982245144936698,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984821100461934,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987677030516543,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991657538186317,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996080396016814,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.119.mlp": [
{
"accuracy": 0.9579520517666089,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9589984478723061,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9638503807547846,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652836777661976,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794134124623317,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9808710330214939,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828689412066811,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898074767552316,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990661756919795,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900479552728173,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911739819340015,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946970295920772,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953726504218617,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970240647855558,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969419331020234,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972586903083873,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983847332799709,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.120.self_attn": [
{
"accuracy": 0.9743526425996893,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9754380714148283,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786222789828715,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833734006081757,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873137353478294,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874622990916434,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923700479496467,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925541396154777,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931788261930802,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939663489104101,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939928324648032,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943409702421999,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994884105901675,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953245549266668,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997177783988024,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976214508673078,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980704418052674,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986836840153525,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994226135482007,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.120.mlp": [
{
"accuracy": 0.9591949476223243,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9601802224979589,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9648128196008896,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9662188214102858,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799275397764224,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814169027009293,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833497904045018,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898931432858502,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907649292267466,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902719581548712,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913522149681261,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947464803086692,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953813909454957,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969285501804399,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971128403895387,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974374978521251,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985742337241965,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.121.self_attn": [
{
"accuracy": 0.9815980605781078,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9824561489451873,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851906553499008,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881623116517929,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914337460833945,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915470563757577,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949940748904881,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951306051623664,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995602759054715,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959025081418651,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958764100814924,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961276287458053,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965489005277816,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968696085025409,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99807267424675,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983737638808394,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986574022454748,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99908194196484,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995721456377252,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.121.mlp": [
{
"accuracy": 0.9587779491159477,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9598118625581264,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9645477711762253,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.966015205375458,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9797957887578952,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813820746678271,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832378053351453,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897114215663781,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906920456131431,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901129217867396,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912501125792531,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946015364441433,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952653040129104,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967355333930371,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967420627117941,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970510237065977,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997971271097317,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.122.self_attn": [
{
"accuracy": 0.982462483507238,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.983275863440021,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860973442463499,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892372322317801,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916947933502103,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918225580630334,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955123696402696,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955584809282109,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959111355710775,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996145395427256,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960804937750494,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963656446381816,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968520602407424,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971057333689379,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982425442186037,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984975716123652,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987838877683604,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991592268833254,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995879039817833,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.122.mlp": [
{
"accuracy": 0.9590376556703919,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9601973477554948,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9645579221021188,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9659039897746161,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.979786815023736,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.981408338601652,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831325269157165,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896374240664667,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904530418203458,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899160028190205,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910926791153064,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945434117199559,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952368437753696,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967568003768591,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969582861828569,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972593947681353,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984229645066846,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.123.self_attn": [
{
"accuracy": 0.9858110149047876,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863535664779576,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885964577674473,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910114516111973,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931800010810165,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932568104153401,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996322825747101,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963883010940137,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966731732221026,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968258982989937,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967125040528021,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969239437854603,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973525099983243,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975544874813702,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985107454765392,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987287306024595,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998996151114912,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992910800700836,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996510329671621,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.123.mlp": [
{
"accuracy": 0.9656366301994574,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9664696559898163,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9707219490879461,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.971989404116022,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.98309657423708,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844109440516484,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860384475047651,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914344424921039,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922339562571755,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917232056981639,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927065581279365,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955263353001914,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961063888561177,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973651964560544,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973649922880883,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976586243198988,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998465704774533,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.124.self_attn": [
{
"accuracy": 0.9859603491464728,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.98643424803097,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887645875446891,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913185048652323,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993230446437864,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933111038500149,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964110522383922,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964805138542464,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967456692829728,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969098312461651,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967619087914691,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969550469081456,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973796985945419,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975972325590096,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985352199107996,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99874226950832,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990089011262171,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992992356297021,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996446222158786,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.124.mlp": [
{
"accuracy": 0.9672127571937285,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9681013941176628,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718316274842149,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729634726322011,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837370199690524,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850790919245858,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865467763064724,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916875514593956,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923381799537885,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918853220971007,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928112092035773,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955817198895506,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.99613256622596,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973785400598947,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975116167487087,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977653981308992,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986578349428448,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.125.self_attn": [
{
"accuracy": 0.9916224641676404,
"total_bits": 320757760,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919178410874385,
"total_bits": 329080832,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932576897799185,
"total_bits": 336024576,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946332312630195,
"total_bits": 401557504,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959398437732536,
"total_bits": 475279360,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959775954852567,
"total_bits": 475479040,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997850799283601,
"total_bits": 609759232,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978920702981812,
"total_bits": 610024448,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980293946205884,
"total_bits": 615020544,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981337781676924,
"total_bits": 623951872,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980350399487897,
"total_bits": 626473984,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981534853271258,
"total_bits": 630355968,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984045170468131,
"total_bits": 637362176,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985289412893747,
"total_bits": 646823936,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990889734196428,
"total_bits": 784740352,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992136205607829,
"total_bits": 797818880,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993735489306824,
"total_bits": 911749120,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995433208073097,
"total_bits": 942718976,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997465551094041,
"total_bits": 1213739008,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.125.mlp": [
{
"accuracy": 0.9710781799727365,
"total_bits": 1575618640,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718735204323342,
"total_bits": 1632503888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753730880390656,
"total_bits": 1823397888,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763523148078668,
"total_bits": 2046482432,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860863950789759,
"total_bits": 2306033760,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872673429352673,
"total_bits": 2368669696,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885248822699252,
"total_bits": 2547224672,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929084884198872,
"total_bits": 2912382048,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935494739023086,
"total_bits": 2955347968,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931537086438191,
"total_bits": 2998880352,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939963133564513,
"total_bits": 3061516288,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963335594358412,
"total_bits": 3691726944,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968203274337085,
"total_bits": 3754362880,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978660938673114,
"total_bits": 4276045920,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980345874240524,
"total_bits": 4425400832,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983096597777483,
"total_bits": 4823859712,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991486073643165,
"total_bits": 5660623360,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.norm.norm": null,
"lm_head.linear": null
},
"last_module_idx": 254
}