| { | |
| "metadata": { | |
| "ParamSize": 245, | |
| "ParamBytes": 1062768640.0, | |
| "BitsPerParam": 4.500610895072402 | |
| }, | |
| "records": [ | |
| { | |
| "dataPath": "params_shard_0.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25956352, | |
| "records": [ | |
| { | |
| "name": "model.layers.0.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.0.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.0.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 2359296 | |
| }, | |
| { | |
| "name": "model.layers.0.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 6553600 | |
| }, | |
| { | |
| "name": "model.layers.0.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 7077888 | |
| }, | |
| { | |
| "name": "model.layers.0.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 7081984 | |
| }, | |
| { | |
| "name": "model.layers.0.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 23859200 | |
| } | |
| ], | |
| "md5sum": "a25496cf4674e9f6408794f4e50908aa" | |
| }, | |
| { | |
| "dataPath": "params_shard_1.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33300480, | |
| "records": [ | |
| { | |
| "name": "model.layers.0.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.0.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.0.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.1.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 9441280 | |
| }, | |
| { | |
| "name": "model.layers.1.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.1.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 11800576 | |
| }, | |
| { | |
| "name": "model.layers.1.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 15994880 | |
| }, | |
| { | |
| "name": "model.layers.1.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 16519168 | |
| }, | |
| { | |
| "name": "model.layers.1.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 16523264 | |
| } | |
| ], | |
| "md5sum": "11cf18c3a0b38b086704550102e0ca49" | |
| }, | |
| { | |
| "dataPath": "params_shard_2.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.10.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "ca71c08215c84d53fc086777bf6951fe" | |
| }, | |
| { | |
| "dataPath": "params_shard_3.bin", | |
| "format": "raw-shard", | |
| "nbytes": 32518144, | |
| "records": [ | |
| { | |
| "name": "model.layers.1.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.1.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.1.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 10485760 | |
| }, | |
| { | |
| "name": "model.layers.1.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 11534336 | |
| }, | |
| { | |
| "name": "model.layers.10.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.10.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 13635584 | |
| }, | |
| { | |
| "name": "model.layers.10.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 13897728 | |
| }, | |
| { | |
| "name": "model.layers.10.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 18092032 | |
| }, | |
| { | |
| "name": "model.layers.10.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 18616320 | |
| }, | |
| { | |
| "name": "model.layers.10.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 18620416 | |
| }, | |
| { | |
| "name": "model.layers.10.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 20717568 | |
| }, | |
| { | |
| "name": "model.layers.10.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 29106176 | |
| }, | |
| { | |
| "name": "model.layers.10.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 30154752 | |
| }, | |
| { | |
| "name": "model.layers.11.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 30158848 | |
| }, | |
| { | |
| "name": "model.layers.11.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 32256000 | |
| } | |
| ], | |
| "md5sum": "6d3deaf75e9ccbbd2c9d20c51408586b" | |
| }, | |
| { | |
| "dataPath": "params_shard_4.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33038336, | |
| "records": [ | |
| { | |
| "name": "model.layers.11.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.11.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 4194304 | |
| }, | |
| { | |
| "name": "model.layers.11.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "model.layers.11.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 4722688 | |
| }, | |
| { | |
| "name": "model.layers.11.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 21499904 | |
| }, | |
| { | |
| "name": "model.layers.11.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 23597056 | |
| }, | |
| { | |
| "name": "model.layers.11.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 31985664 | |
| }, | |
| { | |
| "name": "model.layers.11.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 33034240 | |
| } | |
| ], | |
| "md5sum": "a127e1b009a46ec59c68500380550d47" | |
| }, | |
| { | |
| "dataPath": "params_shard_5.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.2.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0ee2da82994f047cca7774c7ff9a01c5" | |
| }, | |
| { | |
| "dataPath": "params_shard_6.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33034240, | |
| "records": [ | |
| { | |
| "name": "model.layers.12.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.12.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.12.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 2359296 | |
| }, | |
| { | |
| "name": "model.layers.12.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 6553600 | |
| }, | |
| { | |
| "name": "model.layers.12.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 7077888 | |
| }, | |
| { | |
| "name": "model.layers.12.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 23855104 | |
| }, | |
| { | |
| "name": "model.layers.2.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 25952256 | |
| }, | |
| { | |
| "name": "model.layers.2.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 28049408 | |
| }, | |
| { | |
| "name": "model.layers.2.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 28311552 | |
| }, | |
| { | |
| "name": "model.layers.2.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 32505856 | |
| }, | |
| { | |
| "name": "model.layers.2.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 33030144 | |
| } | |
| ], | |
| "md5sum": "baf5749f017706a655f21b9dc35d9590" | |
| }, | |
| { | |
| "dataPath": "params_shard_7.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.3.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "f18e1f5460765d8397529d4333b90afe" | |
| }, | |
| { | |
| "dataPath": "params_shard_8.bin", | |
| "format": "raw-shard", | |
| "nbytes": 32518144, | |
| "records": [ | |
| { | |
| "name": "model.layers.2.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.2.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.2.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 10485760 | |
| }, | |
| { | |
| "name": "model.layers.2.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 11534336 | |
| }, | |
| { | |
| "name": "model.layers.3.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.3.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 13635584 | |
| }, | |
| { | |
| "name": "model.layers.3.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 13897728 | |
| }, | |
| { | |
| "name": "model.layers.3.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 18092032 | |
| }, | |
| { | |
| "name": "model.layers.3.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 18616320 | |
| }, | |
| { | |
| "name": "model.layers.3.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 18620416 | |
| }, | |
| { | |
| "name": "model.layers.3.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 20717568 | |
| }, | |
| { | |
| "name": "model.layers.3.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 29106176 | |
| }, | |
| { | |
| "name": "model.layers.3.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 30154752 | |
| }, | |
| { | |
| "name": "model.layers.4.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 30158848 | |
| }, | |
| { | |
| "name": "model.layers.4.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 32256000 | |
| } | |
| ], | |
| "md5sum": "92a3c7ef838fb432156d8b46e363299f" | |
| }, | |
| { | |
| "dataPath": "params_shard_9.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33038336, | |
| "records": [ | |
| { | |
| "name": "model.layers.4.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.4.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 4194304 | |
| }, | |
| { | |
| "name": "model.layers.4.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "model.layers.4.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 4722688 | |
| }, | |
| { | |
| "name": "model.layers.4.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 21499904 | |
| }, | |
| { | |
| "name": "model.layers.4.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 23597056 | |
| }, | |
| { | |
| "name": "model.layers.4.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 31985664 | |
| }, | |
| { | |
| "name": "model.layers.4.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 33034240 | |
| } | |
| ], | |
| "md5sum": "d5aea50d5fe99aae844914286db3a8ed" | |
| }, | |
| { | |
| "dataPath": "params_shard_10.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25956352, | |
| "records": [ | |
| { | |
| "name": "model.layers.5.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.5.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.5.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 2359296 | |
| }, | |
| { | |
| "name": "model.layers.5.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 6553600 | |
| }, | |
| { | |
| "name": "model.layers.5.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 7077888 | |
| }, | |
| { | |
| "name": "model.layers.5.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 7081984 | |
| }, | |
| { | |
| "name": "model.layers.5.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 23859200 | |
| } | |
| ], | |
| "md5sum": "1e043d657e519fe0e8e8bf1ebf19dbb4" | |
| }, | |
| { | |
| "dataPath": "params_shard_11.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33300480, | |
| "records": [ | |
| { | |
| "name": "model.layers.5.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.5.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.5.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.6.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 9441280 | |
| }, | |
| { | |
| "name": "model.layers.6.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.6.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 11800576 | |
| }, | |
| { | |
| "name": "model.layers.6.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 15994880 | |
| }, | |
| { | |
| "name": "model.layers.6.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 16519168 | |
| }, | |
| { | |
| "name": "model.layers.6.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 16523264 | |
| } | |
| ], | |
| "md5sum": "2f0654640cdb6350fe8ea65ed5ddd112" | |
| }, | |
| { | |
| "dataPath": "params_shard_12.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.7.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "eeb2d5dc7018c7f5cdc7d4fab4fb6065" | |
| }, | |
| { | |
| "dataPath": "params_shard_13.bin", | |
| "format": "raw-shard", | |
| "nbytes": 32518144, | |
| "records": [ | |
| { | |
| "name": "model.layers.6.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.6.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.6.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 10485760 | |
| }, | |
| { | |
| "name": "model.layers.6.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 11534336 | |
| }, | |
| { | |
| "name": "model.layers.7.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.7.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 13635584 | |
| }, | |
| { | |
| "name": "model.layers.7.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 13897728 | |
| }, | |
| { | |
| "name": "model.layers.7.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 18092032 | |
| }, | |
| { | |
| "name": "model.layers.7.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 18616320 | |
| }, | |
| { | |
| "name": "model.layers.7.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 18620416 | |
| }, | |
| { | |
| "name": "model.layers.7.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 20717568 | |
| }, | |
| { | |
| "name": "model.layers.7.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 29106176 | |
| }, | |
| { | |
| "name": "model.layers.7.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 30154752 | |
| }, | |
| { | |
| "name": "model.layers.8.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 30158848 | |
| }, | |
| { | |
| "name": "model.layers.8.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 32256000 | |
| } | |
| ], | |
| "md5sum": "6248d9d4a3e15b049232b58b8f035bcd" | |
| }, | |
| { | |
| "dataPath": "params_shard_14.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33038336, | |
| "records": [ | |
| { | |
| "name": "model.layers.8.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.8.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 4194304 | |
| }, | |
| { | |
| "name": "model.layers.8.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "model.layers.8.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 4722688 | |
| }, | |
| { | |
| "name": "model.layers.8.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 21499904 | |
| }, | |
| { | |
| "name": "model.layers.8.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 23597056 | |
| }, | |
| { | |
| "name": "model.layers.8.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 31985664 | |
| }, | |
| { | |
| "name": "model.layers.8.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 33034240 | |
| } | |
| ], | |
| "md5sum": "6422716453104c4bab4563e604a164f8" | |
| }, | |
| { | |
| "dataPath": "params_shard_15.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25956352, | |
| "records": [ | |
| { | |
| "name": "model.layers.9.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.9.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.9.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 2359296 | |
| }, | |
| { | |
| "name": "model.layers.9.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 6553600 | |
| }, | |
| { | |
| "name": "model.layers.9.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 7077888 | |
| }, | |
| { | |
| "name": "model.layers.9.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 7081984 | |
| }, | |
| { | |
| "name": "model.layers.9.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 23859200 | |
| } | |
| ], | |
| "md5sum": "20b417c323d2aeef269e71f4f86f8ed0" | |
| }, | |
| { | |
| "dataPath": "params_shard_16.bin", | |
| "format": "raw-shard", | |
| "nbytes": 94765056, | |
| "records": [ | |
| { | |
| "name": "model.tok_embeddings.q_weight", | |
| "shape": [ | |
| 92544, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 94765056, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "a73e3b49851508b049183142411b54c4" | |
| }, | |
| { | |
| "dataPath": "params_shard_17.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33091584, | |
| "records": [ | |
| { | |
| "name": "model.layers.9.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.9.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.9.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.tok_embeddings.q_scale", | |
| "shape": [ | |
| 92544, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 11845632, | |
| "byteOffset": 9441280 | |
| }, | |
| { | |
| "name": "model.layers.12.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 21286912 | |
| }, | |
| { | |
| "name": "model.layers.12.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 21291008 | |
| }, | |
| { | |
| "name": "model.layers.12.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 29679616 | |
| }, | |
| { | |
| "name": "model.layers.12.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 30728192 | |
| }, | |
| { | |
| "name": "model.layers.13.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 30732288 | |
| }, | |
| { | |
| "name": "model.layers.13.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 32829440 | |
| } | |
| ], | |
| "md5sum": "5a45d367d750c4b0e369ac5a8ed2b41d" | |
| }, | |
| { | |
| "dataPath": "params_shard_18.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33038336, | |
| "records": [ | |
| { | |
| "name": "model.layers.13.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.13.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 4194304 | |
| }, | |
| { | |
| "name": "model.layers.13.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "model.layers.13.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 4722688 | |
| }, | |
| { | |
| "name": "model.layers.13.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 21499904 | |
| }, | |
| { | |
| "name": "model.layers.13.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 23597056 | |
| }, | |
| { | |
| "name": "model.layers.13.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 31985664 | |
| }, | |
| { | |
| "name": "model.layers.13.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 33034240 | |
| } | |
| ], | |
| "md5sum": "13e4dc09e3ec2748949f7bd5dc727f34" | |
| }, | |
| { | |
| "dataPath": "params_shard_19.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25956352, | |
| "records": [ | |
| { | |
| "name": "model.layers.14.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.14.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.14.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 2359296 | |
| }, | |
| { | |
| "name": "model.layers.14.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 6553600 | |
| }, | |
| { | |
| "name": "model.layers.14.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 7077888 | |
| }, | |
| { | |
| "name": "model.layers.14.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 7081984 | |
| }, | |
| { | |
| "name": "model.layers.14.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 23859200 | |
| } | |
| ], | |
| "md5sum": "f0a9961acf3ba5a5293eaf25b97bec72" | |
| }, | |
| { | |
| "dataPath": "params_shard_20.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33300480, | |
| "records": [ | |
| { | |
| "name": "model.layers.14.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.14.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.14.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.15.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 9441280 | |
| }, | |
| { | |
| "name": "model.layers.15.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.15.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 11800576 | |
| }, | |
| { | |
| "name": "model.layers.15.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 15994880 | |
| }, | |
| { | |
| "name": "model.layers.15.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 16519168 | |
| }, | |
| { | |
| "name": "model.layers.15.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 16523264 | |
| } | |
| ], | |
| "md5sum": "0d1857fe93acd20d78ff83a773fa8f56" | |
| }, | |
| { | |
| "dataPath": "params_shard_21.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.16.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "21ad26a976b5a3426a8902161fcee074" | |
| }, | |
| { | |
| "dataPath": "params_shard_22.bin", | |
| "format": "raw-shard", | |
| "nbytes": 32518144, | |
| "records": [ | |
| { | |
| "name": "model.layers.15.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.15.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.15.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 10485760 | |
| }, | |
| { | |
| "name": "model.layers.15.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 11534336 | |
| }, | |
| { | |
| "name": "model.layers.16.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.16.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 13635584 | |
| }, | |
| { | |
| "name": "model.layers.16.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 13897728 | |
| }, | |
| { | |
| "name": "model.layers.16.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 18092032 | |
| }, | |
| { | |
| "name": "model.layers.16.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 18616320 | |
| }, | |
| { | |
| "name": "model.layers.16.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 18620416 | |
| }, | |
| { | |
| "name": "model.layers.16.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 20717568 | |
| }, | |
| { | |
| "name": "model.layers.16.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 29106176 | |
| }, | |
| { | |
| "name": "model.layers.16.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 30154752 | |
| }, | |
| { | |
| "name": "model.layers.17.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 30158848 | |
| }, | |
| { | |
| "name": "model.layers.17.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 32256000 | |
| } | |
| ], | |
| "md5sum": "53b3bf10b0bfe514abd219ea55f4510b" | |
| }, | |
| { | |
| "dataPath": "params_shard_23.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33038336, | |
| "records": [ | |
| { | |
| "name": "model.layers.17.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.17.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 4194304 | |
| }, | |
| { | |
| "name": "model.layers.17.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "model.layers.17.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 4722688 | |
| }, | |
| { | |
| "name": "model.layers.17.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 21499904 | |
| }, | |
| { | |
| "name": "model.layers.17.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 23597056 | |
| }, | |
| { | |
| "name": "model.layers.17.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 31985664 | |
| }, | |
| { | |
| "name": "model.layers.17.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 33034240 | |
| } | |
| ], | |
| "md5sum": "3e39970d677e0979bc3edd422553e031" | |
| }, | |
| { | |
| "dataPath": "params_shard_24.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25956352, | |
| "records": [ | |
| { | |
| "name": "model.layers.18.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.18.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.18.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 2359296 | |
| }, | |
| { | |
| "name": "model.layers.18.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 6553600 | |
| }, | |
| { | |
| "name": "model.layers.18.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 7077888 | |
| }, | |
| { | |
| "name": "model.layers.18.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 7081984 | |
| }, | |
| { | |
| "name": "model.layers.18.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 23859200 | |
| } | |
| ], | |
| "md5sum": "0622afff97ca80e9b80f33b037f0b8a8" | |
| }, | |
| { | |
| "dataPath": "params_shard_25.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33300480, | |
| "records": [ | |
| { | |
| "name": "model.layers.18.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.18.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.18.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.19.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 9441280 | |
| }, | |
| { | |
| "name": "model.layers.19.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.19.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 11800576 | |
| }, | |
| { | |
| "name": "model.layers.19.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 15994880 | |
| }, | |
| { | |
| "name": "model.layers.19.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 16519168 | |
| }, | |
| { | |
| "name": "model.layers.19.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 16523264 | |
| } | |
| ], | |
| "md5sum": "ff7e5302501238ce8e40772a82042969" | |
| }, | |
| { | |
| "dataPath": "params_shard_26.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.20.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "b570f7e2a24e0b9e5e982b3718752c98" | |
| }, | |
| { | |
| "dataPath": "params_shard_27.bin", | |
| "format": "raw-shard", | |
| "nbytes": 32518144, | |
| "records": [ | |
| { | |
| "name": "model.layers.19.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.19.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.19.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 10485760 | |
| }, | |
| { | |
| "name": "model.layers.19.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 11534336 | |
| }, | |
| { | |
| "name": "model.layers.20.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.20.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 13635584 | |
| }, | |
| { | |
| "name": "model.layers.20.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 13897728 | |
| }, | |
| { | |
| "name": "model.layers.20.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 18092032 | |
| }, | |
| { | |
| "name": "model.layers.20.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 18616320 | |
| }, | |
| { | |
| "name": "model.layers.20.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 18620416 | |
| }, | |
| { | |
| "name": "model.layers.20.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 20717568 | |
| }, | |
| { | |
| "name": "model.layers.20.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 29106176 | |
| }, | |
| { | |
| "name": "model.layers.20.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 30154752 | |
| }, | |
| { | |
| "name": "model.layers.21.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 30158848 | |
| }, | |
| { | |
| "name": "model.layers.21.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 32256000 | |
| } | |
| ], | |
| "md5sum": "d6c40bba2b678bb1ab6c4d2dd511cee9" | |
| }, | |
| { | |
| "dataPath": "params_shard_28.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33038336, | |
| "records": [ | |
| { | |
| "name": "model.layers.21.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.21.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 4194304 | |
| }, | |
| { | |
| "name": "model.layers.21.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 4718592 | |
| }, | |
| { | |
| "name": "model.layers.21.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 4722688 | |
| }, | |
| { | |
| "name": "model.layers.21.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 21499904 | |
| }, | |
| { | |
| "name": "model.layers.21.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 23597056 | |
| }, | |
| { | |
| "name": "model.layers.21.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 31985664 | |
| }, | |
| { | |
| "name": "model.layers.21.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 33034240 | |
| } | |
| ], | |
| "md5sum": "21056282748c3460f741069bc8b376df" | |
| }, | |
| { | |
| "dataPath": "params_shard_29.bin", | |
| "format": "raw-shard", | |
| "nbytes": 25956352, | |
| "records": [ | |
| { | |
| "name": "model.layers.22.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.22.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.22.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 2359296 | |
| }, | |
| { | |
| "name": "model.layers.22.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 6553600 | |
| }, | |
| { | |
| "name": "model.layers.22.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 7077888 | |
| }, | |
| { | |
| "name": "model.layers.22.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 7081984 | |
| }, | |
| { | |
| "name": "model.layers.22.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 23859200 | |
| } | |
| ], | |
| "md5sum": "eb9d208b74ae8e431af56f9744347b8c" | |
| }, | |
| { | |
| "dataPath": "params_shard_30.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33300480, | |
| "records": [ | |
| { | |
| "name": "model.layers.22.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.22.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.22.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.23.attention.wo.q_weight", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 9441280 | |
| }, | |
| { | |
| "name": "model.layers.23.attention.wo.q_scale", | |
| "shape": [ | |
| 2048, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "model.layers.23.attention.wqkv.q_weight", | |
| "shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4194304, | |
| "byteOffset": 11800576 | |
| }, | |
| { | |
| "name": "model.layers.23.attention.wqkv.q_scale", | |
| "shape": [ | |
| 4096, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 15994880 | |
| }, | |
| { | |
| "name": "model.layers.23.attention_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 16519168 | |
| }, | |
| { | |
| "name": "model.layers.23.feed_forward.gate_up_proj.q_weight", | |
| "shape": [ | |
| 16384, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 16523264 | |
| } | |
| ], | |
| "md5sum": "f3e79e9aa185d09a7fda0064826b2e4e" | |
| }, | |
| { | |
| "dataPath": "params_shard_31.bin", | |
| "format": "raw-shard", | |
| "nbytes": 94765056, | |
| "records": [ | |
| { | |
| "name": "output.q_weight", | |
| "shape": [ | |
| 92544, | |
| 256 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 94765056, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "064f77babd659a8e5cffa486e796e41a" | |
| }, | |
| { | |
| "dataPath": "params_shard_32.bin", | |
| "format": "raw-shard", | |
| "nbytes": 23388160, | |
| "records": [ | |
| { | |
| "name": "model.layers.23.feed_forward.gate_up_proj.q_scale", | |
| "shape": [ | |
| 16384, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.23.feed_forward.w2.q_weight", | |
| "shape": [ | |
| 2048, | |
| 1024 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.23.feed_forward.w2.q_scale", | |
| "shape": [ | |
| 2048, | |
| 256 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 10485760 | |
| }, | |
| { | |
| "name": "model.layers.23.ffn_norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 11534336 | |
| }, | |
| { | |
| "name": "model.norm.weight", | |
| "shape": [ | |
| 2048 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 4096, | |
| "byteOffset": 11538432 | |
| }, | |
| { | |
| "name": "output.q_scale", | |
| "shape": [ | |
| 92544, | |
| 64 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 11845632, | |
| "byteOffset": 11542528 | |
| } | |
| ], | |
| "md5sum": "25c3b22e8b1b199b6c3268fcaae64b62" | |
| } | |
| ] | |
| } |