| { | |
| "metadata": { | |
| "ParamSize": 205, | |
| "ParamBytes": 3671255040.0, | |
| "BitsPerParam": 16.0 | |
| }, | |
| "records": [ | |
| { | |
| "dataPath": "params_shard_0.bin", | |
| "format": "raw-shard", | |
| "nbytes": 262144000, | |
| "records": [ | |
| { | |
| "name": "transformer.embd.weight", | |
| "shape": [ | |
| 51200, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144000, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "254b0b3b75b7379a4e1cbd166422d581" | |
| }, | |
| { | |
| "dataPath": "params_shard_1.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "1d86075cab7a17b12bc23b609b0e1a23" | |
| }, | |
| { | |
| "dataPath": "params_shard_2.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0c420ffbfd711e2131f5271c66102718" | |
| }, | |
| { | |
| "dataPath": "params_shard_3.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "a4bc5b186b6a238f1f486bf8cf3ef93c" | |
| }, | |
| { | |
| "dataPath": "params_shard_4.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.1.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "04323ae881c8f62a75f30042883adada" | |
| }, | |
| { | |
| "dataPath": "params_shard_5.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.1.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "7c6565bf96483a607b80cd1b1e11d976" | |
| }, | |
| { | |
| "dataPath": "params_shard_6.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.1.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8a1c55844664e5c92083d16ded362b0c" | |
| }, | |
| { | |
| "dataPath": "params_shard_7.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "98d4e8967f14837a29db76190ab143ab" | |
| }, | |
| { | |
| "dataPath": "params_shard_8.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26342400, | |
| "records": [ | |
| { | |
| "name": "transformer.h.0.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.0.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 15360 | |
| }, | |
| { | |
| "name": "transformer.h.0.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13122560 | |
| }, | |
| { | |
| "name": "transformer.h.0.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13127680 | |
| }, | |
| { | |
| "name": "transformer.h.0.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.0.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13153280 | |
| }, | |
| { | |
| "name": "transformer.h.0.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13158400 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13178880 | |
| }, | |
| { | |
| "name": "transformer.h.1.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26286080 | |
| }, | |
| { | |
| "name": "transformer.h.1.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26291200 | |
| }, | |
| { | |
| "name": "transformer.h.1.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26311680 | |
| }, | |
| { | |
| "name": "transformer.h.1.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "transformer.h.1.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26321920 | |
| }, | |
| { | |
| "name": "transformer.h.2.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26327040 | |
| } | |
| ], | |
| "md5sum": "329750ff6e5b7163357f147fbe437a69" | |
| }, | |
| { | |
| "dataPath": "params_shard_9.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "92c7a3dac1b0e4c178d695b8de429e98" | |
| }, | |
| { | |
| "dataPath": "params_shard_10.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0f31d37e0d8d3fcf837f86a7a8c684fc" | |
| }, | |
| { | |
| "dataPath": "params_shard_11.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "a6ed72fb47e10f1b5a4c2754c373b7f6" | |
| }, | |
| { | |
| "dataPath": "params_shard_12.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "ec594a40413cf2a2d646b6957e5d03d7" | |
| }, | |
| { | |
| "dataPath": "params_shard_13.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.3.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3ec56e021be377c39b6c0b116bb35628" | |
| }, | |
| { | |
| "dataPath": "params_shard_14.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "b082066a6093ab82e5332b22ecce2ff2" | |
| }, | |
| { | |
| "dataPath": "params_shard_15.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.2.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.2.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.2.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.2.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.2.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.2.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.3.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.3.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.3.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.3.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.3.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.3.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.3.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "transformer.h.4.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26311680 | |
| } | |
| ], | |
| "md5sum": "219af8ce693d196d660d6327124fd6a2" | |
| }, | |
| { | |
| "dataPath": "params_shard_16.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8738ac6db7d61471ae5b40bfc66270aa" | |
| }, | |
| { | |
| "dataPath": "params_shard_17.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "043a62b5c63b81462096ffa61d7e5b2a" | |
| }, | |
| { | |
| "dataPath": "params_shard_18.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "00b46be5cc0bcd86c30cd605a5301545" | |
| }, | |
| { | |
| "dataPath": "params_shard_19.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "84e61313f6653d2acec2f206d3dd0a7e" | |
| }, | |
| { | |
| "dataPath": "params_shard_20.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.5.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3bf00c9c71502d112f0c5daafa7f4459" | |
| }, | |
| { | |
| "dataPath": "params_shard_21.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8d32b3f5ea1c6fc504364fac2ddafc36" | |
| }, | |
| { | |
| "dataPath": "params_shard_22.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.4.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.4.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.4.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.4.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.4.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.4.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.5.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.5.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.5.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.5.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.5.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.5.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.5.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "transformer.h.6.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26311680 | |
| } | |
| ], | |
| "md5sum": "3ac6772168ff1c95f73ebc54359eff86" | |
| }, | |
| { | |
| "dataPath": "params_shard_23.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "75da9ae456af72601f8fb85adea7d67a" | |
| }, | |
| { | |
| "dataPath": "params_shard_24.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "15613cc3936c6078dea5be7abb33e697" | |
| }, | |
| { | |
| "dataPath": "params_shard_25.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "b1fd7e449fc7911d19c93fd6afda1240" | |
| }, | |
| { | |
| "dataPath": "params_shard_26.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "ee00368c240a6d66c041f4c30528c544" | |
| }, | |
| { | |
| "dataPath": "params_shard_27.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.7.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "61031af848adf766d88beec5d9f0acb0" | |
| }, | |
| { | |
| "dataPath": "params_shard_28.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "6ffb8e934fc2a9d0c34f79f70b96fe80" | |
| }, | |
| { | |
| "dataPath": "params_shard_29.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.6.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.6.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.6.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.6.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.6.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.6.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.7.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.7.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.7.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.7.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.7.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.7.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.7.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "transformer.h.8.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26311680 | |
| } | |
| ], | |
| "md5sum": "10b259f7b80334935ea33f9f0abdb01d" | |
| }, | |
| { | |
| "dataPath": "params_shard_30.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "eb6126e1ec26e7cc1178ff32aa79ae2d" | |
| }, | |
| { | |
| "dataPath": "params_shard_31.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "849b5b23fe85f1a319c4d777da925b8a" | |
| }, | |
| { | |
| "dataPath": "params_shard_32.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "86f8b08366dee01d027a7103e71946f0" | |
| }, | |
| { | |
| "dataPath": "params_shard_33.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "cd5d8fcd207089ee8ac08233d37ab831" | |
| }, | |
| { | |
| "dataPath": "params_shard_34.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.9.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "d68a0eaa4eafc7ff64b28ee55dc0842c" | |
| }, | |
| { | |
| "dataPath": "params_shard_35.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "fbf9982d6f0e89131bddc19cb49c1347" | |
| }, | |
| { | |
| "dataPath": "params_shard_36.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.8.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.8.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.8.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.8.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.8.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.8.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.9.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.9.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.9.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.9.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.9.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.9.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.9.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "transformer.h.10.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26311680 | |
| } | |
| ], | |
| "md5sum": "266231919f5eaf3a31bfd0cea392b498" | |
| }, | |
| { | |
| "dataPath": "params_shard_37.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "690d2340a046f40907fe625d1d751d19" | |
| }, | |
| { | |
| "dataPath": "params_shard_38.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "c2ec913e8df76d589ca8d160dfd402db" | |
| }, | |
| { | |
| "dataPath": "params_shard_39.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.11.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "68f4929215e67c28364d58d6843c001b" | |
| }, | |
| { | |
| "dataPath": "params_shard_40.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.11.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "6c884087fbdbb1e779cd0e1093659625" | |
| }, | |
| { | |
| "dataPath": "params_shard_41.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.11.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "98f5d251641571e4235cfd91909f6e3a" | |
| }, | |
| { | |
| "dataPath": "params_shard_42.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "48d8412bcfbfd85027ed63f9dcdd6873" | |
| }, | |
| { | |
| "dataPath": "params_shard_43.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.10.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.10.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.10.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.10.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.10.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.10.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.11.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.11.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.11.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.11.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.11.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "transformer.h.12.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26311680 | |
| } | |
| ], | |
| "md5sum": "0d90e5ac45d7ce00bedbb54a0c688cda" | |
| }, | |
| { | |
| "dataPath": "params_shard_44.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "b7af2eed401eea46d396ac51d3acf274" | |
| }, | |
| { | |
| "dataPath": "params_shard_45.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "96d1f7f7778192d9542442eee48c735e" | |
| }, | |
| { | |
| "dataPath": "params_shard_46.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.13.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "d79d7c20fcdcb92a9b956a0a9a48026d" | |
| }, | |
| { | |
| "dataPath": "params_shard_47.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.13.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0d391b68314789a69d6c76e3c65c3538" | |
| }, | |
| { | |
| "dataPath": "params_shard_48.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.13.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0fbde014ac32a62e1d189f38ed4d0afe" | |
| }, | |
| { | |
| "dataPath": "params_shard_49.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "98f570ce4858df7ca19475d32695dce9" | |
| }, | |
| { | |
| "dataPath": "params_shard_50.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.12.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.12.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.12.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.12.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.12.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.12.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.13.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.13.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.13.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.13.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.13.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "transformer.h.14.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26311680 | |
| } | |
| ], | |
| "md5sum": "92d2c8113e788af21045a243cdb7325c" | |
| }, | |
| { | |
| "dataPath": "params_shard_51.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "75b97abbacf685e51794e7256d610e3c" | |
| }, | |
| { | |
| "dataPath": "params_shard_52.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "1d3e0557ab7aa60cbd6acd33c5070327" | |
| }, | |
| { | |
| "dataPath": "params_shard_53.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.15.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "5e3feeee08b04e7af3a715c10b291db0" | |
| }, | |
| { | |
| "dataPath": "params_shard_54.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.15.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "16783d3af4c068f88250e69b0588e17e" | |
| }, | |
| { | |
| "dataPath": "params_shard_55.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.15.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "9333c61853fb4d63c389aba68ae5ad39" | |
| }, | |
| { | |
| "dataPath": "params_shard_56.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "84d79f00c414566b46f1e620b31dc30b" | |
| }, | |
| { | |
| "dataPath": "params_shard_57.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.14.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.14.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.14.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.14.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.14.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.14.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.15.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.15.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.15.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.15.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.15.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "transformer.h.16.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26311680 | |
| } | |
| ], | |
| "md5sum": "6b2bb3e094b2ad81f48990b9b0d8e830" | |
| }, | |
| { | |
| "dataPath": "params_shard_58.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "e1248c1b0c1ebd29dcbd6eb7f4ec338a" | |
| }, | |
| { | |
| "dataPath": "params_shard_59.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "040407b43bf3ff9aa561fbe29148c18b" | |
| }, | |
| { | |
| "dataPath": "params_shard_60.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.17.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "720112a08108adab7ab733f50cbffeb3" | |
| }, | |
| { | |
| "dataPath": "params_shard_61.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.17.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "3ae5a6ae85945525cb84d302a8026236" | |
| }, | |
| { | |
| "dataPath": "params_shard_62.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.17.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "effb14500be7ce716373c4ec3239acf1" | |
| }, | |
| { | |
| "dataPath": "params_shard_63.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "792b23c0059afdc26cf6ffdd82b84d00" | |
| }, | |
| { | |
| "dataPath": "params_shard_64.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26327040, | |
| "records": [ | |
| { | |
| "name": "transformer.h.16.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.16.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.16.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.16.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.16.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.16.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.17.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.17.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.17.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.17.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.17.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "transformer.h.18.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 26311680 | |
| } | |
| ], | |
| "md5sum": "12e1f05efa6bb94064fb5910224468ea" | |
| }, | |
| { | |
| "dataPath": "params_shard_65.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8d7270666cc389cd116199ef3ad130ed" | |
| }, | |
| { | |
| "dataPath": "params_shard_66.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "9597fde475154730ecb8429a52ee37b6" | |
| }, | |
| { | |
| "dataPath": "params_shard_67.bin", | |
| "format": "raw-shard", | |
| "nbytes": 39321600, | |
| "records": [ | |
| { | |
| "name": "transformer.h.19.mixer.Wqkv.weight", | |
| "shape": [ | |
| 7680, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 39321600, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "6d270b3ed05c6e4948e4099af7cfbd09" | |
| }, | |
| { | |
| "dataPath": "params_shard_68.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.19.mlp.fc1.weight", | |
| "shape": [ | |
| 10240, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "f2eb64215c23166b166ca5b7d9bc7223" | |
| }, | |
| { | |
| "dataPath": "params_shard_69.bin", | |
| "format": "raw-shard", | |
| "nbytes": 52428800, | |
| "records": [ | |
| { | |
| "name": "transformer.h.19.mlp.fc2.weight", | |
| "shape": [ | |
| 2560, | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 52428800, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "1f2d6952a50b443abe22e14084353243" | |
| }, | |
| { | |
| "dataPath": "params_shard_70.bin", | |
| "format": "raw-shard", | |
| "nbytes": 262144000, | |
| "records": [ | |
| { | |
| "name": "lm_head.linear.weight", | |
| "shape": [ | |
| 51200, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 262144000, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "8feed96b8b5f10f70c5a66f554136517" | |
| }, | |
| { | |
| "dataPath": "params_shard_71.bin", | |
| "format": "raw-shard", | |
| "nbytes": 26424320, | |
| "records": [ | |
| { | |
| "name": "transformer.h.18.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "transformer.h.18.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13107200 | |
| }, | |
| { | |
| "name": "transformer.h.18.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 13112320 | |
| }, | |
| { | |
| "name": "transformer.h.18.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13132800 | |
| }, | |
| { | |
| "name": "transformer.h.18.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13137920 | |
| }, | |
| { | |
| "name": "transformer.h.18.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 13143040 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.Wqkv.bias", | |
| "shape": [ | |
| 7680 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 15360, | |
| "byteOffset": 13148160 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.out_proj.weight", | |
| "shape": [ | |
| 2560, | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 13107200, | |
| "byteOffset": 13163520 | |
| }, | |
| { | |
| "name": "transformer.h.19.mixer.out_proj.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26270720 | |
| }, | |
| { | |
| "name": "transformer.h.19.mlp.fc1.bias", | |
| "shape": [ | |
| 10240 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 20480, | |
| "byteOffset": 26275840 | |
| }, | |
| { | |
| "name": "transformer.h.19.mlp.fc2.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26296320 | |
| }, | |
| { | |
| "name": "transformer.h.19.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26301440 | |
| }, | |
| { | |
| "name": "transformer.h.19.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26306560 | |
| }, | |
| { | |
| "name": "lm_head.ln.weight", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26311680 | |
| }, | |
| { | |
| "name": "lm_head.ln.bias", | |
| "shape": [ | |
| 2560 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 5120, | |
| "byteOffset": 26316800 | |
| }, | |
| { | |
| "name": "lm_head.linear.bias", | |
| "shape": [ | |
| 51200 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 102400, | |
| "byteOffset": 26321920 | |
| } | |
| ], | |
| "md5sum": "22ef24edb6636c86c84357b430fdc866" | |
| } | |
| ] | |
| } |