diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,3975 @@ +{ + "metadata": { + "ParamSize": 313, + "ParamBytes": 3927297024.0, + "BitsPerParam": 4.12551973205239 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 272498688, + "records": [ + { + "name": "lm_head.q_weight", + "shape": [ + 448, + 152064 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 272498688, + "byteOffset": 0 + } + ], + "md5sum": "91512a2ccc889bbc2973fbaa56196313" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.22.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "66a56df31655e06ac8564ec30bb22523" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.23.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "fc3be3ae6556543212acd60095e4ed27" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "47935df9300c4ecae39816a83cdcb093" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.24.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "5a488f5cfca98ec4deb7563bda119af9" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.24.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "95e1db326fd4db9eafe794303d15c7a8" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 31127552, + "records": [ + { + "name": "lm_head.q_scale", + "shape": [ + 28, + 152064 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8515584, + "byteOffset": 0 + }, + { + "name": "model.layers.22.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 8515584 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9576448 + }, + { + "name": "model.layers.23.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 9583616 + }, + { + "name": "model.layers.23.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 10644480 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 12766208 + }, + { + "name": "model.layers.23.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 12773376 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 12782592 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 21040128 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 21298176 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 27720704 + }, + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 27921408 + }, + { + "name": "model.layers.24.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 27928576 + }, + { + "name": "model.layers.24.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 28989440 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 31111168 + }, + { + "name": "model.layers.24.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 31118336 + } + ], + "md5sum": "a84895cb465880d241578fab58f5d595" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.25.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "842c549980a1bdd101b859ad4f1d7926" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.25.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "8ac068e4a5e4299309e49930e46097ef" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "c602f1cf720da0692cd96ccbfeb41f89" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.24.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.24.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.25.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.25.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.25.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "3a34b06dfa7ae1134a960029af40cc37" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.26.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "da32f4d63f66637aea79dc758a587bbb" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.27.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "cb4953bc6efbf26254e924e8e130bb7b" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "8d5c364c6640d7fc700d18fa44f44246" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.26.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.26.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.27.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.27.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.27.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "800bef530003d17b8b1ee9407af03467" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 272498688, + "records": [ + { + "name": "model.embed_tokens.q_weight", + "shape": [ + 152064, + 448 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 272498688, + "byteOffset": 0 + } + ], + "md5sum": "651b41d47ddbad57cfd24496ba877a49" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.0.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "cb9fb3cb8cd346d264458e786aa66ee8" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.0.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "214da31fef487ece04d771d894c202ce" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.1.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "2cf294a8d28a6f6fe7ddb35f5d2add6d" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 33498112, + "records": [ + { + "name": "model.layers.27.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.norm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.embed_tokens.q_scale", + "shape": [ + 152064, + 28 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8515584, + "byteOffset": 6630400 + }, + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15145984 + }, + { + "name": "model.layers.0.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15153152 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16214016 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18335744 + }, + { + "name": "model.layers.0.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18342912 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18352128 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26609664 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26867712 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33290240 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33490944 + } + ], + "md5sum": "09b5a45f40a560368107fde9b1097a47" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.1.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "db89e7a2545fb7e7395d77f4bea533ad" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.2.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "3f47a5418ab5e3bfe2d2367ebd119b5a" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.2.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "d85befae5d692f95508ef83bdc76a3ee" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.1.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.1.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.2.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.2.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "ce4d11f6f1797cec52a4aac8423d91d4" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.3.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "3778a0e27a061012dd69f66932d6491d" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "b4d3af9b63e5a3f8d9dfa26e5ff908a5" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.4.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "2e1afb581d4d9a273f0f727aec05d956" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.4.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "f411154f5c3a68fdf34936014774a593" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.2.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.3.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.3.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.3.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.4.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.4.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "3665ad1402d9efaf298187d182d8df4d" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.5.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "a8ea2827e176ac24c32ae5d1aa543281" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.5.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "9b4b9857238720ccc04ea20779433660" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 33507328, + "records": [ + { + "name": "model.layers.4.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.5.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.5.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33490944 + }, + { + "name": "model.layers.6.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 33498112 + } + ], + "md5sum": "e92b50041c31f190f6ef37aead60289f" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.10.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "a7bbe494d06be2865b8d45b8ce5b4918" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.10.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "d4aba73e76d9484a949718086592524c" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.11.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "01f9abb30ae224368ab6eab0d8d9e548" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.6.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.10.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.10.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "67fde5b7decbf00d5f52f0621a0a6a3a" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.11.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "bb0962519016a40d8568015fd6a64a82" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.12.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "d7a7336039e0c5b699a5b3114412982f" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.12.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "304298762c86751d719de7435fdf699a" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.11.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.11.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.12.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.12.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "34671cfbe4c54bf7e9e695446222cc20" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.13.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "a185d6d6d533ff78986f91b02f483832" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.13.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "21754932f47a0fe0c168209f22d047a6" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.14.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "fde5907eac095dbab584e75471b23d93" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 27113472, + "records": [ + { + "name": "model.layers.12.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.13.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.13.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 24975360 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 27097088 + }, + { + "name": "model.layers.14.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 27104256 + } + ], + "md5sum": "3046dfe61606c92ac84adedde52d2d78" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.6.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "ac806b205f1ef2ab2f96b1c0be16c47a" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.6.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "a09c10f8ad2392a16f2a4e8bad7b09b2" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.7.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "970b77458444162ecbaa00c6fce058e3" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "0b7293284a390cfdcc17b81167051140" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 30043136, + "records": [ + { + "name": "model.layers.14.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.6.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15138816 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16199680 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18321408 + }, + { + "name": "model.layers.7.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18328576 + }, + { + "name": "model.layers.7.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19389440 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21511168 + }, + { + "name": "model.layers.7.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21518336 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21527552 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29785088 + } + ], + "md5sum": "9ff5bfaf0fd3c2ef03e6c24b2c6fb29b" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.8.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "d019721fc7cd58544730c0055e9e5301" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "7cb0943afa9584924ebd91dfab56d9f4" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.9.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "d191e55e4b5441b888ab600103eb4d95" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.9.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "59d14ec37335f4816c859054deed0b7e" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.7.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.8.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.8.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.8.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.9.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.9.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "6fc8beb2c8b49117a4f82dc198bd82ce" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.14.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "0ca41bbf534a4f55685f2bab9ec44ae7" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.15.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "640ea093f5ab9167a2a2ffaa4e5a119e" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "1ba17b97c6ca5709435e99b176593fe1" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 27921408, + "records": [ + { + "name": "model.layers.9.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.14.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15138816 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 16199680 + }, + { + "name": "model.layers.15.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 16206848 + }, + { + "name": "model.layers.15.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 17267712 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 19389440 + }, + { + "name": "model.layers.15.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 19396608 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 19405824 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 27663360 + } + ], + "md5sum": "84817c91491c0ee12426d7d6322d28dc" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.16.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "d8b5d1382748051b8932ece0cca40d6a" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "91db88e3419248907619c9bada30ac89" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.17.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "959803fd2c912f00ff9ca30edbaf2bc1" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.17.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "3bf0bb25d3e43775a68d80b4f2d864d9" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 28174336, + "records": [ + { + "name": "model.layers.15.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.16.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.16.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.16.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.17.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 24975360 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 26036224 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 28157952 + }, + { + "name": "model.layers.17.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 28165120 + } + ], + "md5sum": "5c3e7c374a4fa7c65fe2536467214ff1" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.18.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "8f0c7961a5c4f054838ef56c8a8db22e" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.18.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "ae31d2a920e1252aee9f1017d4f71919" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.19.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "2298e48ea0d32a87f477f2653f7f5996" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 33490944, + "records": [ + { + "name": "model.layers.17.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 15138816 + }, + { + "name": "model.layers.18.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 15145984 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 16206848 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18328576 + }, + { + "name": "model.layers.18.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 18335744 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 18344960 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 26602496 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 26860544 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 33283072 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 33483776 + } + ], + "md5sum": "33283e1d3fd3bdcff0ed5d1bdb46f9bd" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "97b095d3f075767eb764b76c9bc8ed2f" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.20.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "edfe6e952c105421551c8be66e01632d" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.20.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "4698bba78442d82280c532c12f1e0199" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 30059520, + "records": [ + { + "name": "model.layers.19.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 0 + }, + { + "name": "model.layers.19.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 1060864 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 3182592 + }, + { + "name": "model.layers.19.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 3189760 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 3198976 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 11456512 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 11714560 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 18137088 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 18337792 + }, + { + "name": "model.layers.20.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 18344960 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 19405824 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 21527552 + }, + { + "name": "model.layers.20.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 21534720 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 21543936 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 29801472 + } + ], + "md5sum": "bc86c294cd852aa4f60a23d168620822" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 33947648, + "records": [ + { + "name": "model.layers.21.mlp.down_proj.q_weight", + "shape": [ + 2368, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 33947648, + "byteOffset": 0 + } + ], + "md5sum": "3e75a6273baa195a90421409b0020f2e" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.21.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "65178e31e9f5b3acce857134af81771d" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 67895296, + "records": [ + { + "name": "model.layers.22.mlp.gate_up_proj.q_weight", + "shape": [ + 448, + 37888 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 67895296, + "byteOffset": 0 + } + ], + "md5sum": "32f58a92739ec3cc0e9720b4e1fcbd45" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 27113472, + "records": [ + { + "name": "model.layers.20.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 0 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 6422528 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 6623232 + }, + { + "name": "model.layers.21.mlp.down_proj.q_scale", + "shape": [ + 148, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1060864, + "byteOffset": 6630400 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 7691264 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 9812992 + }, + { + "name": "model.layers.21.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 9820160 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 9829376 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 18086912 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 18344960 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 24767488 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 24968192 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_scale", + "shape": [ + 28, + 37888 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2121728, + "byteOffset": 24975360 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 7168, + "byteOffset": 27097088 + }, + { + "name": "model.layers.22.self_attn.c_attn.bias", + "shape": [ + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 9216, + "byteOffset": 27104256 + } + ], + "md5sum": "042c78dabdfdc8ec9d961b8357b7bf6d" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 15138816, + "records": [ + { + "name": "model.layers.22.self_attn.c_attn.q_weight", + "shape": [ + 448, + 4608 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8257536, + "byteOffset": 0 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_scale", + "shape": [ + 28, + 4608 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 258048, + "byteOffset": 8257536 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_weight", + "shape": [ + 448, + 3584 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6422528, + "byteOffset": 8515584 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_scale", + "shape": [ + 28, + 3584 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 200704, + "byteOffset": 14938112 + } + ], + "md5sum": "fe671144827d5b034c4450300ca97e83" + } + ] +} \ No newline at end of file