diff --git "a/ndarray-cache-b16.json" "b/ndarray-cache-b16.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache-b16.json" @@ -0,0 +1,3851 @@ +{ + "metadata": { + "ParamSize": 343, + "ParamBytes": 888430592.0, + "BitsPerParam": 5.011345602622326 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 52428800, + "records": [ + { + "name": "transformer.embd.q_weight", + "shape": [ + 51200, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 52428800, + "byteOffset": 0 + } + ], + "md5sum": "481da31cb16684c311ac9c4d12d259f4" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 25468928, + "records": [ + { + "name": "transformer.embd.q_scale", + "shape": [ + 51200, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6553600, + "byteOffset": 0 + }, + { + "name": "transformer.h.0.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 6553600 + }, + { + "name": "transformer.h.0.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 6557696 + }, + { + "name": "transformer.h.0.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 6561792 + }, + { + "name": "transformer.h.0.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 12853248 + }, + { + "name": "transformer.h.0.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 13639680 + }, + { + "name": "transformer.h.0.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 13651968 + }, + { + "name": "transformer.h.0.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 15749120 + }, + { + "name": "transformer.h.0.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 16011264 + }, + { + "name": "transformer.h.0.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 16015360 + }, + { + "name": "transformer.h.0.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 24403968 + }, + { + "name": "transformer.h.0.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 25452544 + } + ], + "md5sum": "6d3bbadb7cfc1ce6670cdd040f6cc218" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.0.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.0.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.0.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.1.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.1.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.1.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.1.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.1.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.1.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.1.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.1.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.1.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.1.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.1.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "9faebf36cbeb3cddacdddf3517fffb1c" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.1.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.1.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.1.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.2.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.2.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.2.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.2.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.2.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.2.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.2.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.2.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.2.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.2.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.2.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "f7e00cd244f4a482f2aa3d6eeaee5ec7" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.2.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.2.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.2.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.3.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.3.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.3.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.3.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.3.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.3.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.3.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.3.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.3.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.3.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.3.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "68e406b28b1d0d8d3ab5b75f2707d1fc" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.3.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.3.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.3.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.4.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.4.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.4.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.4.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.4.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.4.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.4.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.4.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.4.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.4.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.4.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "ace361649b7076f5b5bd542c12952e76" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.4.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.4.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.4.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.5.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.5.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.5.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.5.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.5.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.5.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.5.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.5.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.5.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.5.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.5.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "1a53bb04148d7bc211505496bdc2079a" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.5.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.5.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.5.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.6.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.6.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.6.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.6.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.6.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.6.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.6.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.6.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.6.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.6.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.6.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "d194e035094589f57760fdafbea49ff1" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.6.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.6.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.6.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.7.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.7.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.7.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.7.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.7.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.7.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.7.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.7.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.7.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.7.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.7.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "2d34543415f07093371084b99a6422ef" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.7.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.7.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.7.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.8.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.8.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.8.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.8.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.8.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.8.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.8.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.8.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.8.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.8.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.8.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "03522374a69510e1db15faa61d00739f" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.8.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.8.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.8.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.9.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.9.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.9.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.9.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.9.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.9.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.9.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.9.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.9.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.9.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.9.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "0ffec31e8626b7b954136d47143a774b" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.9.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.9.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.9.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.10.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.10.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.10.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.10.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.10.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.10.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.10.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.10.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.10.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.10.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.10.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "5f56e72335c437ab0765fa7bc29a4738" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.10.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.10.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.10.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.11.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.11.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.11.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.11.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.11.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.11.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.11.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.11.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.11.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.11.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.11.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "0fa4d57d6ca48afdf6e64b0529933091" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.11.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.11.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.11.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.12.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.12.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.12.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.12.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.12.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.12.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.12.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.12.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.12.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.12.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.12.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "903b6cb4e709f162d3312dc616afb5f5" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.12.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.12.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.12.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.13.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.13.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.13.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.13.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.13.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.13.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.13.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.13.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.13.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.13.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.13.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "5e678f4db6cf536b35118b9a9af2d5bf" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.13.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.13.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.13.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.14.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.14.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.14.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.14.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.14.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.14.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.14.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.14.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.14.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.14.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.14.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "1053259ddb2a5119efaa362430b6b20a" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.14.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.14.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.14.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.15.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.15.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.15.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.15.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.15.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.15.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.15.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.15.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.15.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.15.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.15.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "55aa2471b3917bc87a56ed0a95fdb2c0" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.15.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.15.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.15.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.16.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.16.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.16.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.16.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.16.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.16.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.16.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.16.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.16.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.16.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.16.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "03628a26cbb1d9615f1424b876ff7b07" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.16.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.16.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.16.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.17.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.17.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.17.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.17.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.17.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.17.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.17.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.17.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.17.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.17.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.17.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "f90ed1c5a9f5c7c718632e5888a26965" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.17.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.17.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.17.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.18.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.18.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.18.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.18.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.18.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.18.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.18.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.18.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.18.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.18.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.18.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "5bbfa545db0d3db69833d5ed6c5d8baf" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.18.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.18.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.18.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.19.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.19.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.19.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.19.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.19.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.19.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.19.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.19.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.19.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.19.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.19.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "93215d09f486cd20d5b2c6292df1a612" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.19.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.19.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.19.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.20.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.20.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.20.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.20.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.20.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.20.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.20.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.20.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.20.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.20.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.20.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "8adf156777fd3dfdbccf7465cfadfbd4" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.20.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.20.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.20.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.21.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.21.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.21.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.21.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.21.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.21.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.21.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.21.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.21.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.21.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.21.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "1cce39030d60c018bba55a4c47c2249f" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.21.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.21.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.21.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.22.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.22.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.22.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.22.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.22.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.22.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.22.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.22.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.22.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.22.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.22.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "e00507b0e8322073d6b499b53438a0f2" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 28356608, + "records": [ + { + "name": "transformer.h.22.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.22.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.22.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "transformer.h.23.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "transformer.h.23.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "transformer.h.23.mixer.Wqkv.q_weight", + "shape": [ + 6144, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 9449472 + }, + { + "name": "transformer.h.23.mixer.Wqkv.q_scale", + "shape": [ + 6144, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 786432, + "byteOffset": 15740928 + }, + { + "name": "transformer.h.23.mixer.Wqkv.bias", + "shape": [ + 6144 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 12288, + "byteOffset": 16527360 + }, + { + "name": "transformer.h.23.mixer.out_proj.q_weight", + "shape": [ + 2048, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16539648 + }, + { + "name": "transformer.h.23.mixer.out_proj.q_scale", + "shape": [ + 2048, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 262144, + "byteOffset": 18636800 + }, + { + "name": "transformer.h.23.mixer.out_proj.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 18898944 + }, + { + "name": "transformer.h.23.mlp.fc1.q_weight", + "shape": [ + 8192, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 18903040 + }, + { + "name": "transformer.h.23.mlp.fc1.q_scale", + "shape": [ + 8192, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 27291648 + }, + { + "name": "transformer.h.23.mlp.fc1.bias", + "shape": [ + 8192 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 16384, + "byteOffset": 28340224 + } + ], + "md5sum": "73e50a7b6303594671db3ee30265d3ff" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 52428800, + "records": [ + { + "name": "lm_head.linear.q_weight", + "shape": [ + 51200, + 256 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 52428800, + "byteOffset": 0 + } + ], + "md5sum": "3f7e0086f7b85998fddaea74cecf2338" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 16105472, + "records": [ + { + "name": "transformer.h.23.mlp.fc2.q_weight", + "shape": [ + 2048, + 1024 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "transformer.h.23.mlp.fc2.q_scale", + "shape": [ + 2048, + 256 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "transformer.h.23.mlp.fc2.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9437184 + }, + { + "name": "lm_head.ln.weight", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9441280 + }, + { + "name": "lm_head.ln.bias", + "shape": [ + 2048 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 4096, + "byteOffset": 9445376 + }, + { + "name": "lm_head.linear.q_scale", + "shape": [ + 51200, + 64 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 6553600, + "byteOffset": 9449472 + }, + { + "name": "lm_head.linear.bias", + "shape": [ + 51200 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 102400, + "byteOffset": 16003072 + } + ], + "md5sum": "bada4e74772fd12c7bcc21b50b51a93a" + } + ] +} \ No newline at end of file