diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,3975 @@ +{ + "metadata": { + "ParamSize": 313, + "ParamBytes": 3927297024.0, + "BitsPerParam": 4.12551973205239 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 272498688, + "records": [ + { + "name": "lm_head.q_weight", + "shape": [ + 448, + 152064 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 272498688, + "byteOffset": 0 + } + ], + "md5sum": "91512a2ccc889bbc2973fbaa56196313" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.22.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "8f0b258f8e84d647914d31bcc1696b6b" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.23.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "58fab4f8e356d1d0c40dc9b92ce157f6" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "37b4a2cb483737a76b3ddf2b732ba35d" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.24.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "4dea7409e326de98b000bce588ec8b08" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.24.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "f931f0f84c6cb4285eede4b591936012" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 31127552, + "records": [ + { + "name": "lm_head.q_scale", + "shape": [ + 28, + 152064 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8515584, + "byteOffset": 0 + }, + { + "name": "model.layers.22.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 8515584 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9576448 + }, + { + "name": "model.layers.23.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 9583616 + }, + { + "name": "model.layers.23.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 10644480 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 12766208 + }, + { + "name": "model.layers.23.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 12773376 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 12782592 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 21040128 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 21298176 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 27720704 + }, + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 27921408 + }, + { + "name": "model.layers.24.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 27928576 + }, + { + "name": "model.layers.24.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 28989440 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 31111168 + }, + { + "name": "model.layers.24.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 31118336 + } + ], + "md5sum": "86122dfc64fa8699afc044af1cb6e037" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.25.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "84144a16b763fdba0f69599ffe603768" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.25.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "78fba423a49b3a91aeb9a499a5bdb0c3" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "3adcea82dae5a9b840d9963c2d9a9027" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.24.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.24.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.25.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.25.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.25.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "cd9f6a0d561f90b69cd935456290e9d5" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.26.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "11f52fd11b0af3a0c9dbe679c87ebcc4" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.27.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "fec513254b9b4f84cee882c050864f02" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "89dc87db1dc006748f80fb2dd86682e9" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.26.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.26.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.27.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.27.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.27.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "3e19f80aa347756ebd8ee9bf2ba4de3c" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 272498688, + "records": [ + { + "name": "model.embed_tokens.q_weight", + "shape": [ + 152064, + 448 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 272498688, + "byteOffset": 0 + } + ], + "md5sum": "651b41d47ddbad57cfd24496ba877a49" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.0.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "42bf7284af2b81ece19367b59996fd33" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.0.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "016b39fc23240622f64ce3a4d71a3da3" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.1.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "e7f61010e38037d8b3a4a73a65ca4f27" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 33498112, + "records": [ + { + "name": "model.layers.27.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.norm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.embed_tokens.q_scale", + "shape": [ + 152064, + 28 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8515584, + "byteOffset": 6630400 + }, + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15145984 + }, + { + "name": "model.layers.0.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15153152 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16214016 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18335744 + }, + { + "name": "model.layers.0.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18342912 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18352128 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26609664 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26867712 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33290240 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33490944 + } + ], + "md5sum": "77096f09de550bebc42bf626e964d707" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.1.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "b54fad214e56e4e5579f988186777cb6" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.2.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "eec9fdbe06cce2e877474255fbf41e80" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.2.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "9b5c12605ef430633cabb6e2b9075da0" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.1.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.1.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.2.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.2.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "6e11060cf52b457f0d9cf43ba2847618" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.3.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "663b1739d3e3c2d1cae4a42dcd0b8c7c" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "7384c335cc1e88776d3bc955f2257f26" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.4.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "989165f4d3b353c87b0e4d4a2253482c" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.4.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "d015f345de9f887abf4548e22dbddfb2" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.2.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.3.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.3.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.3.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.4.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.4.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "363c9cc907b7d1fc8097572291b60cb1" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.5.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "94fa2535b36cf52b51927fa56675e9be" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.5.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "4af8ff689f990b43acce043ff1768191" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 33507328, + "records": [ + { + "name": "model.layers.4.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.5.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.5.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33490944 + }, + { + "name": "model.layers.6.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 33498112 + } + ], + "md5sum": "63f2ddd9850cf60acf114fa182e6e6e5" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.10.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "7b4d8f5094df549f11314b4475a2d514" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.10.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "8ad5e1b8d744e6d6f84c82df4e4cc607" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.11.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "8c065ee11fda7a54bb6b45bc66cb9482" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.6.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.10.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.10.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "b38fb4c41d1a6bd53044c7cc38d3e2ef" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.11.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "48a3e25f4204e577880a82cbbaa22670" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.12.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "0aef2c8170e6431fe51cc09df19c2117" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.12.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "47b6a4e595205346d0cd61808470f42b" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.11.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.11.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.12.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.12.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "58aa7b0093de4996dac5ebfb41f66a17" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.13.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "f259abfcbb5b8cad262448b12fe33d1a" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.13.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "a72958ba499200b98ab9e72cee32950a" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.14.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "6297e47019b2c5f70d06b94a7467d919" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 27113472, + "records": [ + { + "name": "model.layers.12.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.13.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.13.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 24975360 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 27097088 + }, + { + "name": "model.layers.14.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 27104256 + } + ], + "md5sum": "062590c45bb4c9e26a1f14ae7eee9abd" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.6.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "9783ca7303e90311c5400048fa42f848" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.6.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "70e2ce9335a36422e64fa651b0065f9b" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.7.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "dece616767b12c5be8dd41da88a0b4f1" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "668df82977eceb82e4ae6ca8420d3626" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 30043136, + "records": [ + { + "name": "model.layers.14.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.6.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15138816 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16199680 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18321408 + }, + { + "name": "model.layers.7.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18328576 + }, + { + "name": "model.layers.7.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19389440 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21511168 + }, + { + "name": "model.layers.7.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21518336 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21527552 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29785088 + } + ], + "md5sum": "42497b1ebbe4756dd0fc58cdf46a87d5" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.8.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "783789a7e81ab0f24a2ef56ed23f2aca" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "f7f2d693b298669fff5f14c0bbc336bc" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.9.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "7d9c89d62556d60021ba0308ff9a953e" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.9.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "1514604945278feb8fb67306fc23e48e" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.7.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.8.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.8.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.8.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.9.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.9.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "60fd2929ea0cc59be22d9243bdb645e1" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.14.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "dd9d755a4a860dbfc075e2a6ccfec8df" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.15.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "db773282067475e2f5fc5e98dd8a16c3" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "ad7d6ee36f41c7085a5486034f195af7" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 27921408, + "records": [ + { + "name": "model.layers.9.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.14.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15138816 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 16199680 + }, + { + "name": "model.layers.15.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 16206848 + }, + { + "name": "model.layers.15.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 17267712 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 19389440 + }, + { + "name": "model.layers.15.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 19396608 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 19405824 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 27663360 + } + ], + "md5sum": "c213651019716a1c96c47da9d7ac87cf" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.16.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "809e30501fb734b87438121dd134999a" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "bb8690c382f7421821d8df546e6927a8" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.17.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "81cb88a910234dea8557b98696ad028c" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.17.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "7b3e2083906c4678117ef0a811271f17" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.15.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.16.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.16.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.16.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.17.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.17.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "5b9308646ba7f1daeb6c593fd37a04a0" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.18.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "8ff61386a59d54ff6a6249d20b79ec45" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.18.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "7cd01eeaeb9963fd8a61f77997aa5e6e" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.19.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "a572c2dfbf8deaf23e6667a4dfe62f19" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.17.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.18.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.18.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "6ab459d73ae15370ec32ac35de213dd2" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "31d2f63cd4d1e9c4156d0f046c7ec9d0" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.20.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "eabc0f0bed5acdd80ef28c774230dc9d" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.20.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "63e8d690fd6077b847f11c2c6b965fa7" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.19.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.19.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.19.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.20.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.20.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "ef7be3d3f74052b3629228e535620eb1" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.21.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "812d0c49096b226712413a2a45415008" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.21.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "33c31ed74f771c01ce59176fd3976050" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.22.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "cbc3691b2acf9edb629cc6911d45768a" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 27113472, + "records": [ + { + "name": "model.layers.20.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.21.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.21.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 24975360 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 27097088 + }, + { + "name": "model.layers.22.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 27104256 + } + ], + "md5sum": "abd2bbeb6190dbac23eb1c2eb327bb17" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 15138816, + "records": [ + { + "name": "model.layers.22.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + } + ], + "md5sum": "b9d1823dea3d231a8e4f99503fa70bc9" + } + ] +} \ No newline at end of file