{ "metadata": { "ParamSize": 183, "ParamBytes": 1409830912.0, "BitsPerParam": 4.500347711112945 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 262144000, "records": [ { "name": "model.embed_tokens.q_weight", "shape": [ 256000, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 262144000, "byteOffset": 0 } ], "md5sum": "4500be88779baf43abfc8597bf8cf1fe" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 16777216, "records": [ { "name": "model.layers.0.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 0 } ], "md5sum": "11b6476bf39b9f23d5fed8b859335a3c" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 32772096, "records": [ { "name": "model.embed_tokens.q_scale", "shape": [ 256000, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 32768000, "byteOffset": 0 }, { "name": "model.layers.0.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 32768000 } ], "md5sum": "f379b208c95b8217c994896c7b6f2498" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.0.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "811b87af8acf881a27c966fc2dbcc866" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.1.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "99ba5de36ee07b36b94c654f98154e91" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 30482432, "records": [ { "name": "model.layers.0.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 0 }, { "name": "model.layers.0.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 2097152 }, { "name": "model.layers.0.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 6291456 }, { "name": "model.layers.0.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 6295552 }, { "name": "model.layers.0.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 8916992 }, { "name": "model.layers.0.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 9244672 }, { "name": "model.layers.0.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11341824 }, { "name": "model.layers.1.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 11603968 }, { "name": "model.layers.1.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 11608064 }, { "name": "model.layers.1.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 28385280 } ], "md5sum": "8765790dd5ba60df1f3e7ca3cd44a823" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.10.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "96628a52b99c0b92e9d4b4ca006b9d27" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 32583680, "records": [ { "name": "model.layers.1.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 0 }, { "name": "model.layers.1.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 4194304 }, { "name": "model.layers.1.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 4198400 }, { "name": "model.layers.1.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 6819840 }, { "name": "model.layers.1.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 7147520 }, { "name": "model.layers.1.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 9244672 }, { "name": "model.layers.10.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9506816 }, { "name": "model.layers.10.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 9510912 }, { "name": "model.layers.10.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26288128 }, { "name": "model.layers.10.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 28385280 }, { "name": "model.layers.10.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 32579584 } ], "md5sum": "3d7d87a03da4abd28973fa45b8422840" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.11.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "17fff7aec80df3c609bdb930cf1ca3ef" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 33431552, "records": [ { "name": "model.layers.10.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 0 }, { "name": "model.layers.10.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 2621440 }, { "name": "model.layers.10.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2949120 }, { "name": "model.layers.10.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 5046272 }, { "name": "model.layers.11.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 5308416 }, { "name": "model.layers.11.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 5312512 }, { "name": "model.layers.11.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22089728 }, { "name": "model.layers.11.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 24186880 }, { "name": "model.layers.11.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 28381184 }, { "name": "model.layers.11.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 28385280 }, { "name": "model.layers.11.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 31006720 }, { "name": "model.layers.11.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 31334400 } ], "md5sum": "a4b995ea31754819ec61ed94e020ae1c" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.12.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "fbce69d99fff816998a4641b7456c520" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 16777216, "records": [ { "name": "model.layers.13.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 0 } ], "md5sum": "b7233843a23e5112bbc1422f7161fc0d" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.13.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "9af497a0d5d22872cd67f10128e570c5" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 30748672, "records": [ { "name": "model.layers.11.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 0 }, { "name": "model.layers.12.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 262144 }, { "name": "model.layers.12.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 266240 }, { "name": "model.layers.12.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 17043456 }, { "name": "model.layers.12.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 19140608 }, { "name": "model.layers.12.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 23334912 }, { "name": "model.layers.12.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 23339008 }, { "name": "model.layers.12.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 25960448 }, { "name": "model.layers.12.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26288128 }, { "name": "model.layers.12.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 28385280 }, { "name": "model.layers.13.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 28647424 }, { "name": "model.layers.13.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 28651520 } ], "md5sum": "bbb67f351dde779ef8f33ec90875bd33" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.14.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "f394533aec140c012d6aba09331778c4" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 32583680, "records": [ { "name": "model.layers.13.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 0 }, { "name": "model.layers.13.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 4194304 }, { "name": "model.layers.13.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 4198400 }, { "name": "model.layers.13.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 6819840 }, { "name": "model.layers.13.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 7147520 }, { "name": "model.layers.13.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 9244672 }, { "name": "model.layers.14.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9506816 }, { "name": "model.layers.14.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 9510912 }, { "name": "model.layers.14.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26288128 }, { "name": "model.layers.14.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 28385280 }, { "name": "model.layers.14.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 32579584 } ], "md5sum": "7d171cdc31ed69961f5e79a286d06aa6" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.15.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "de486ff25a393f3dcdb0bf8d5758fd28" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 33431552, "records": [ { "name": "model.layers.14.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 0 }, { "name": "model.layers.14.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 2621440 }, { "name": "model.layers.14.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2949120 }, { "name": "model.layers.14.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 5046272 }, { "name": "model.layers.15.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 5308416 }, { "name": "model.layers.15.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 5312512 }, { "name": "model.layers.15.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22089728 }, { "name": "model.layers.15.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 24186880 }, { "name": "model.layers.15.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 28381184 }, { "name": "model.layers.15.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 28385280 }, { "name": "model.layers.15.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 31006720 }, { "name": "model.layers.15.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 31334400 } ], "md5sum": "0686c57209ff99cc9bf88ba08563b478" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.16.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "b23041c42c00d1bb4e2a9bf2f71720d9" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.17.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "0ac04fca4cce82f427e016e9774cceef" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 32841728, "records": [ { "name": "model.layers.15.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 0 }, { "name": "model.layers.16.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 262144 }, { "name": "model.layers.16.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 266240 }, { "name": "model.layers.16.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 17043456 }, { "name": "model.layers.16.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 19140608 }, { "name": "model.layers.16.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 23334912 }, { "name": "model.layers.16.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 23339008 }, { "name": "model.layers.16.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 25960448 }, { "name": "model.layers.16.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26288128 }, { "name": "model.layers.16.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 28385280 }, { "name": "model.layers.17.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 28647424 } ], "md5sum": "024de43a50d63440ad744e3034cde06d" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.2.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "d9cfc40db0870d1424ae6bfb8292a2b8" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 33431552, "records": [ { "name": "model.layers.17.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 0 }, { "name": "model.layers.17.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 2621440 }, { "name": "model.layers.17.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2949120 }, { "name": "model.layers.17.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 5046272 }, { "name": "model.layers.2.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 5308416 }, { "name": "model.layers.2.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 5312512 }, { "name": "model.layers.2.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22089728 }, { "name": "model.layers.2.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 24186880 }, { "name": "model.layers.2.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 28381184 }, { "name": "model.layers.2.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 28385280 }, { "name": "model.layers.2.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 31006720 }, { "name": "model.layers.2.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 31334400 } ], "md5sum": "36d4575512b8a3b13afdeb3af4000208" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.3.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "a65325f51ca1bf40a6be33cc9cd33019" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 16777216, "records": [ { "name": "model.layers.4.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 0 } ], "md5sum": "3fcfa3796de77684d00e83b226954073" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.4.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "c23e574d1ca278094721a99879139444" }, { "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 30748672, "records": [ { "name": "model.layers.2.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 0 }, { "name": "model.layers.3.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 262144 }, { "name": "model.layers.3.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 266240 }, { "name": "model.layers.3.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 17043456 }, { "name": "model.layers.3.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 19140608 }, { "name": "model.layers.3.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 23334912 }, { "name": "model.layers.3.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 23339008 }, { "name": "model.layers.3.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 25960448 }, { "name": "model.layers.3.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26288128 }, { "name": "model.layers.3.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 28385280 }, { "name": "model.layers.4.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 28647424 }, { "name": "model.layers.4.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 28651520 } ], "md5sum": "53d652513cf5f426f67a2f084c619b81" }, { "dataPath": "params_shard_27.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.5.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "023d2b6a65d7a065f5d8b734e0902511" }, { "dataPath": "params_shard_28.bin", "format": "raw-shard", "nbytes": 32583680, "records": [ { "name": "model.layers.4.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 0 }, { "name": "model.layers.4.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 4194304 }, { "name": "model.layers.4.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 4198400 }, { "name": "model.layers.4.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 6819840 }, { "name": "model.layers.4.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 7147520 }, { "name": "model.layers.4.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 9244672 }, { "name": "model.layers.5.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9506816 }, { "name": "model.layers.5.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 9510912 }, { "name": "model.layers.5.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26288128 }, { "name": "model.layers.5.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 28385280 }, { "name": "model.layers.5.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 32579584 } ], "md5sum": "104f7fb347a93eba3d8b6e8f5858e4a3" }, { "dataPath": "params_shard_29.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.6.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "8f64f3942a8e10c689489510ebdf7e3c" }, { "dataPath": "params_shard_30.bin", "format": "raw-shard", "nbytes": 33431552, "records": [ { "name": "model.layers.5.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 0 }, { "name": "model.layers.5.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 2621440 }, { "name": "model.layers.5.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2949120 }, { "name": "model.layers.5.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 5046272 }, { "name": "model.layers.6.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 5308416 }, { "name": "model.layers.6.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 5312512 }, { "name": "model.layers.6.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22089728 }, { "name": "model.layers.6.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 24186880 }, { "name": "model.layers.6.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 28381184 }, { "name": "model.layers.6.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 28385280 }, { "name": "model.layers.6.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 31006720 }, { "name": "model.layers.6.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 31334400 } ], "md5sum": "91dd99cb8d5dcdb06e83f3671ebb7a10" }, { "dataPath": "params_shard_31.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.7.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "27ae58e11a4585b357ee0364a952c1c7" }, { "dataPath": "params_shard_32.bin", "format": "raw-shard", "nbytes": 16777216, "records": [ { "name": "model.layers.8.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 0 } ], "md5sum": "e70f33478f73dad46b9edabf478017b9" }, { "dataPath": "params_shard_33.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.8.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "6df9bd91f6838bfee2f7e45938ffa0ba" }, { "dataPath": "params_shard_34.bin", "format": "raw-shard", "nbytes": 30748672, "records": [ { "name": "model.layers.6.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 0 }, { "name": "model.layers.7.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 262144 }, { "name": "model.layers.7.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 266240 }, { "name": "model.layers.7.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 17043456 }, { "name": "model.layers.7.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 19140608 }, { "name": "model.layers.7.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 23334912 }, { "name": "model.layers.7.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 23339008 }, { "name": "model.layers.7.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 25960448 }, { "name": "model.layers.7.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26288128 }, { "name": "model.layers.7.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 28385280 }, { "name": "model.layers.8.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 28647424 }, { "name": "model.layers.8.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 28651520 } ], "md5sum": "7c4551ab8b0db64d06b105ca63752e41" }, { "dataPath": "params_shard_35.bin", "format": "raw-shard", "nbytes": 33554432, "records": [ { "name": "model.layers.9.mlp.gate_up_proj.q_weight", "shape": [ 32768, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 33554432, "byteOffset": 0 } ], "md5sum": "994c6d0d88e52dfb84d07e2986a1890c" }, { "dataPath": "params_shard_36.bin", "format": "raw-shard", "nbytes": 32583680, "records": [ { "name": "model.layers.8.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 0 }, { "name": "model.layers.8.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 4194304 }, { "name": "model.layers.8.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 4198400 }, { "name": "model.layers.8.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 6819840 }, { "name": "model.layers.8.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 7147520 }, { "name": "model.layers.8.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 9244672 }, { "name": "model.layers.9.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 9506816 }, { "name": "model.layers.9.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 9510912 }, { "name": "model.layers.9.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 26288128 }, { "name": "model.layers.9.mlp.gate_up_proj.q_scale", "shape": [ 32768, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4194304, "byteOffset": 28385280 }, { "name": "model.layers.9.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 32579584 } ], "md5sum": "1c7774514abceeff4f57aaf934e05f94" }, { "dataPath": "params_shard_37.bin", "format": "raw-shard", "nbytes": 24195072, "records": [ { "name": "model.layers.9.self_attn.qkv_proj.q_weight", "shape": [ 2560, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2621440, "byteOffset": 0 }, { "name": "model.layers.9.self_attn.qkv_proj.q_scale", "shape": [ 2560, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 327680, "byteOffset": 2621440 }, { "name": "model.layers.9.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 2949120 }, { "name": "model.layers.9.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 5046272 }, { "name": "model.layers.17.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 5308416 }, { "name": "model.layers.17.mlp.down_proj.q_weight", "shape": [ 2048, 2048 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 16777216, "byteOffset": 5312512 }, { "name": "model.layers.17.mlp.down_proj.q_scale", "shape": [ 2048, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22089728 }, { "name": "model.layers.17.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 24186880 }, { "name": "model.norm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 24190976 } ], "md5sum": "912e83dc9b08d17d803870c7fc011394" } ] }