{"format": "freetoken_weight", "version": 1, "align": 4096, "shard_limit": 8589934592, "total_bytes": 157460918272, "tensors": [{"name": "embed.weight", "kind": "weight", "dtype": "bfloat16", "shape": [129280, 4096], "global_off": 0, "nbytes": 1059061760}, {"name": "norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 1059061760, "nbytes": 8192}, {"name": "head", "kind": "weight", "dtype": "bfloat16", "shape": [129280, 4096], "global_off": 1059069952, "nbytes": 1059061760}, {"name": "hc_head_fn", "kind": "weight", "dtype": "float32", "shape": [4, 16384], "global_off": 2118131712, "nbytes": 262144}, {"name": "hc_head_base", "kind": "weight", "dtype": "float32", "shape": [4], "global_off": 2118393856, "nbytes": 16}, {"name": "hc_head_scale", "kind": "weight", "dtype": "float32", "shape": [1], "global_off": 2118397952, "nbytes": 4}, {"name": "layers.0.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 2118402048, "nbytes": 4194304}, {"name": "layers.0.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 2122596352, "nbytes": 256}, {"name": "layers.0.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 2122600448, "nbytes": 2048}, {"name": "layers.0.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 2122604544, "nbytes": 33554432}, {"name": "layers.0.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 2156158976, "nbytes": 2048}, {"name": "layers.0.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 2156163072, "nbytes": 2097152}, {"name": "layers.0.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 2158260224, "nbytes": 128}, {"name": "layers.0.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 2158264320, "nbytes": 1024}, {"name": "layers.0.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 2158268416, "nbytes": 67108864}, {"name": "layers.0.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 2225377280, "nbytes": 33554432}, {"name": "layers.0.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 2258931712, "nbytes": 2048}, {"name": "layers.0.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 2258935808, "nbytes": 256}, {"name": "layers.0.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 2258939904, "nbytes": 8192}, {"name": "layers.0.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 2258948096, "nbytes": 8192}, {"name": "layers.0.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 2258956288, "nbytes": 2097152}, {"name": "layers.0.ffn.gate.tid2eid", "kind": "weight", "dtype": "int64", "shape": [129280, 6], "global_off": 2261053440, "nbytes": 6205440}, {"name": "layers.0.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 2267258880, "nbytes": 8388608}, {"name": "layers.0.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 2275647488, "nbytes": 512}, {"name": "layers.0.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 2275651584, "nbytes": 8388608}, {"name": "layers.0.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 2284040192, "nbytes": 512}, {"name": "layers.0.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 2284044288, "nbytes": 8388608}, {"name": "layers.0.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 2292432896, "nbytes": 512}, {"name": "layers.0.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 2292436992, "nbytes": 1572864}, {"name": "layers.0.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 2294009856, "nbytes": 1572864}, {"name": "layers.0.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 2295582720, "nbytes": 96}, {"name": "layers.0.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 2295586816, "nbytes": 96}, {"name": "layers.0.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 2295590912, "nbytes": 12}, {"name": "layers.0.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 2295595008, "nbytes": 12}, {"name": "layers.1.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 2295599104, "nbytes": 4194304}, {"name": "layers.1.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 2299793408, "nbytes": 256}, {"name": "layers.1.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 2299797504, "nbytes": 2048}, {"name": "layers.1.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 2299801600, "nbytes": 33554432}, {"name": "layers.1.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 2333356032, "nbytes": 2048}, {"name": "layers.1.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 2333360128, "nbytes": 2097152}, {"name": "layers.1.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 2335457280, "nbytes": 128}, {"name": "layers.1.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 2335461376, "nbytes": 1024}, {"name": "layers.1.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 2335465472, "nbytes": 67108864}, {"name": "layers.1.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 2402574336, "nbytes": 33554432}, {"name": "layers.1.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 2436128768, "nbytes": 2048}, {"name": "layers.1.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 2436132864, "nbytes": 256}, {"name": "layers.1.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 2436136960, "nbytes": 8192}, {"name": "layers.1.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 2436145152, "nbytes": 8192}, {"name": "layers.1.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 2436153344, "nbytes": 2097152}, {"name": "layers.1.ffn.gate.tid2eid", "kind": "weight", "dtype": "int64", "shape": [129280, 6], "global_off": 2438250496, "nbytes": 6205440}, {"name": "layers.1.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 2444455936, "nbytes": 8388608}, {"name": "layers.1.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 2452844544, "nbytes": 512}, {"name": "layers.1.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 2452848640, "nbytes": 8388608}, {"name": "layers.1.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 2461237248, "nbytes": 512}, {"name": "layers.1.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 2461241344, "nbytes": 8388608}, {"name": "layers.1.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 2469629952, "nbytes": 512}, {"name": "layers.1.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 2469634048, "nbytes": 1572864}, {"name": "layers.1.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 2471206912, "nbytes": 1572864}, {"name": "layers.1.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 2472779776, "nbytes": 96}, {"name": "layers.1.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 2472783872, "nbytes": 96}, {"name": "layers.1.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 2472787968, "nbytes": 12}, {"name": "layers.1.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 2472792064, "nbytes": 12}, {"name": "layers.2.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 2472796160, "nbytes": 4194304}, {"name": "layers.2.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 2476990464, "nbytes": 256}, {"name": "layers.2.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 2476994560, "nbytes": 2048}, {"name": "layers.2.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 2476998656, "nbytes": 33554432}, {"name": "layers.2.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 2510553088, "nbytes": 2048}, {"name": "layers.2.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 2510557184, "nbytes": 2097152}, {"name": "layers.2.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 2512654336, "nbytes": 128}, {"name": "layers.2.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 2512658432, "nbytes": 1024}, {"name": "layers.2.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 2512662528, "nbytes": 67108864}, {"name": "layers.2.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 2579771392, "nbytes": 33554432}, {"name": "layers.2.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 2613325824, "nbytes": 2048}, {"name": "layers.2.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 2613329920, "nbytes": 256}, {"name": "layers.2.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 2613334016, "nbytes": 16384}, {"name": "layers.2.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 2613350400, "nbytes": 8388608}, {"name": "layers.2.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 2621739008, "nbytes": 8388608}, {"name": "layers.2.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 2630127616, "nbytes": 1024}, {"name": "layers.2.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 2630131712, "nbytes": 8388608}, {"name": "layers.2.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 2638520320, "nbytes": 512}, {"name": "layers.2.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 2638524416, "nbytes": 524288}, {"name": "layers.2.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 2639048704, "nbytes": 4096}, {"name": "layers.2.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 2639052800, "nbytes": 2097152}, {"name": "layers.2.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 2641149952, "nbytes": 2097152}, {"name": "layers.2.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 2643247104, "nbytes": 256}, {"name": "layers.2.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 2643251200, "nbytes": 8192}, {"name": "layers.2.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 2643259392, "nbytes": 8192}, {"name": "layers.2.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 2643267584, "nbytes": 2097152}, {"name": "layers.2.ffn.gate.tid2eid", "kind": "weight", "dtype": "int64", "shape": [129280, 6], "global_off": 2645364736, "nbytes": 6205440}, {"name": "layers.2.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 2651570176, "nbytes": 8388608}, {"name": "layers.2.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 2659958784, "nbytes": 512}, {"name": "layers.2.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 2659962880, "nbytes": 8388608}, {"name": "layers.2.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 2668351488, "nbytes": 512}, {"name": "layers.2.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 2668355584, "nbytes": 8388608}, {"name": "layers.2.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 2676744192, "nbytes": 512}, {"name": "layers.2.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 2676748288, "nbytes": 1572864}, {"name": "layers.2.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 2678321152, "nbytes": 1572864}, {"name": "layers.2.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 2679894016, "nbytes": 96}, {"name": "layers.2.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 2679898112, "nbytes": 96}, {"name": "layers.2.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 2679902208, "nbytes": 12}, {"name": "layers.2.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 2679906304, "nbytes": 12}, {"name": "layers.3.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 2679910400, "nbytes": 4194304}, {"name": "layers.3.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 2684104704, "nbytes": 256}, {"name": "layers.3.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 2684108800, "nbytes": 2048}, {"name": "layers.3.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 2684112896, "nbytes": 33554432}, {"name": "layers.3.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 2717667328, "nbytes": 2048}, {"name": "layers.3.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 2717671424, "nbytes": 2097152}, {"name": "layers.3.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 2719768576, "nbytes": 128}, {"name": "layers.3.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 2719772672, "nbytes": 1024}, {"name": "layers.3.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 2719776768, "nbytes": 67108864}, {"name": "layers.3.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 2786885632, "nbytes": 33554432}, {"name": "layers.3.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 2820440064, "nbytes": 2048}, {"name": "layers.3.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 2820444160, "nbytes": 256}, {"name": "layers.3.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 2820448256, "nbytes": 262144}, {"name": "layers.3.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 2820710400, "nbytes": 4194304}, {"name": "layers.3.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 2824904704, "nbytes": 4194304}, {"name": "layers.3.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 2829099008, "nbytes": 1024}, {"name": "layers.3.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 2829103104, "nbytes": 8192}, {"name": "layers.3.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 2829111296, "nbytes": 8192}, {"name": "layers.3.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 2829119488, "nbytes": 2097152}, {"name": "layers.3.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 2831216640, "nbytes": 1024}, {"name": "layers.3.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 2831220736, "nbytes": 8388608}, {"name": "layers.3.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 2839609344, "nbytes": 512}, {"name": "layers.3.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 2839613440, "nbytes": 8388608}, {"name": "layers.3.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 2848002048, "nbytes": 512}, {"name": "layers.3.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 2848006144, "nbytes": 8388608}, {"name": "layers.3.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 2856394752, "nbytes": 512}, {"name": "layers.3.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 2856398848, "nbytes": 1572864}, {"name": "layers.3.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 2857971712, "nbytes": 1572864}, {"name": "layers.3.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 2859544576, "nbytes": 96}, {"name": "layers.3.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 2859548672, "nbytes": 96}, {"name": "layers.3.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 2859552768, "nbytes": 12}, {"name": "layers.3.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 2859556864, "nbytes": 12}, {"name": "layers.4.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 2859560960, "nbytes": 4194304}, {"name": "layers.4.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 2863755264, "nbytes": 256}, {"name": "layers.4.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 2863759360, "nbytes": 2048}, {"name": "layers.4.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 2863763456, "nbytes": 33554432}, {"name": "layers.4.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 2897317888, "nbytes": 2048}, {"name": "layers.4.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 2897321984, "nbytes": 2097152}, {"name": "layers.4.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 2899419136, "nbytes": 128}, {"name": "layers.4.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 2899423232, "nbytes": 1024}, {"name": "layers.4.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 2899427328, "nbytes": 67108864}, {"name": "layers.4.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 2966536192, "nbytes": 33554432}, {"name": "layers.4.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 3000090624, "nbytes": 2048}, {"name": "layers.4.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 3000094720, "nbytes": 256}, {"name": "layers.4.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 3000098816, "nbytes": 16384}, {"name": "layers.4.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 3000115200, "nbytes": 8388608}, {"name": "layers.4.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 3008503808, "nbytes": 8388608}, {"name": "layers.4.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3016892416, "nbytes": 1024}, {"name": "layers.4.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 3016896512, "nbytes": 8388608}, {"name": "layers.4.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 3025285120, "nbytes": 512}, {"name": "layers.4.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 3025289216, "nbytes": 524288}, {"name": "layers.4.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 3025813504, "nbytes": 4096}, {"name": "layers.4.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3025817600, "nbytes": 2097152}, {"name": "layers.4.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3027914752, "nbytes": 2097152}, {"name": "layers.4.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3030011904, "nbytes": 256}, {"name": "layers.4.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3030016000, "nbytes": 8192}, {"name": "layers.4.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3030024192, "nbytes": 8192}, {"name": "layers.4.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3030032384, "nbytes": 2097152}, {"name": "layers.4.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 3032129536, "nbytes": 1024}, {"name": "layers.4.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3032133632, "nbytes": 8388608}, {"name": "layers.4.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3040522240, "nbytes": 512}, {"name": "layers.4.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 3040526336, "nbytes": 8388608}, {"name": "layers.4.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 3048914944, "nbytes": 512}, {"name": "layers.4.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3048919040, "nbytes": 8388608}, {"name": "layers.4.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3057307648, "nbytes": 512}, {"name": "layers.4.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3057311744, "nbytes": 1572864}, {"name": "layers.4.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3058884608, "nbytes": 1572864}, {"name": "layers.4.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3060457472, "nbytes": 96}, {"name": "layers.4.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3060461568, "nbytes": 96}, {"name": "layers.4.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3060465664, "nbytes": 12}, {"name": "layers.4.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3060469760, "nbytes": 12}, {"name": "layers.5.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 3060473856, "nbytes": 4194304}, {"name": "layers.5.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 3064668160, "nbytes": 256}, {"name": "layers.5.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 3064672256, "nbytes": 2048}, {"name": "layers.5.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 3064676352, "nbytes": 33554432}, {"name": "layers.5.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 3098230784, "nbytes": 2048}, {"name": "layers.5.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 3098234880, "nbytes": 2097152}, {"name": "layers.5.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 3100332032, "nbytes": 128}, {"name": "layers.5.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3100336128, "nbytes": 1024}, {"name": "layers.5.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 3100340224, "nbytes": 67108864}, {"name": "layers.5.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 3167449088, "nbytes": 33554432}, {"name": "layers.5.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 3201003520, "nbytes": 2048}, {"name": "layers.5.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 3201007616, "nbytes": 256}, {"name": "layers.5.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 3201011712, "nbytes": 262144}, {"name": "layers.5.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 3201273856, "nbytes": 4194304}, {"name": "layers.5.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 3205468160, "nbytes": 4194304}, {"name": "layers.5.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3209662464, "nbytes": 1024}, {"name": "layers.5.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3209666560, "nbytes": 8192}, {"name": "layers.5.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3209674752, "nbytes": 8192}, {"name": "layers.5.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3209682944, "nbytes": 2097152}, {"name": "layers.5.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 3211780096, "nbytes": 1024}, {"name": "layers.5.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3211784192, "nbytes": 8388608}, {"name": "layers.5.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3220172800, "nbytes": 512}, {"name": "layers.5.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 3220176896, "nbytes": 8388608}, {"name": "layers.5.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 3228565504, "nbytes": 512}, {"name": "layers.5.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3228569600, "nbytes": 8388608}, {"name": "layers.5.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3236958208, "nbytes": 512}, {"name": "layers.5.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3236962304, "nbytes": 1572864}, {"name": "layers.5.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3238535168, "nbytes": 1572864}, {"name": "layers.5.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3240108032, "nbytes": 96}, {"name": "layers.5.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3240112128, "nbytes": 96}, {"name": "layers.5.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3240116224, "nbytes": 12}, {"name": "layers.5.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3240120320, "nbytes": 12}, {"name": "layers.6.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 3240124416, "nbytes": 4194304}, {"name": "layers.6.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 3244318720, "nbytes": 256}, {"name": "layers.6.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 3244322816, "nbytes": 2048}, {"name": "layers.6.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 3244326912, "nbytes": 33554432}, {"name": "layers.6.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 3277881344, "nbytes": 2048}, {"name": "layers.6.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 3277885440, "nbytes": 2097152}, {"name": "layers.6.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 3279982592, "nbytes": 128}, {"name": "layers.6.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3279986688, "nbytes": 1024}, {"name": "layers.6.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 3279990784, "nbytes": 67108864}, {"name": "layers.6.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 3347099648, "nbytes": 33554432}, {"name": "layers.6.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 3380654080, "nbytes": 2048}, {"name": "layers.6.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 3380658176, "nbytes": 256}, {"name": "layers.6.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 3380662272, "nbytes": 16384}, {"name": "layers.6.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 3380678656, "nbytes": 8388608}, {"name": "layers.6.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 3389067264, "nbytes": 8388608}, {"name": "layers.6.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3397455872, "nbytes": 1024}, {"name": "layers.6.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 3397459968, "nbytes": 8388608}, {"name": "layers.6.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 3405848576, "nbytes": 512}, {"name": "layers.6.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 3405852672, "nbytes": 524288}, {"name": "layers.6.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 3406376960, "nbytes": 4096}, {"name": "layers.6.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3406381056, "nbytes": 2097152}, {"name": "layers.6.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3408478208, "nbytes": 2097152}, {"name": "layers.6.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3410575360, "nbytes": 256}, {"name": "layers.6.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3410579456, "nbytes": 8192}, {"name": "layers.6.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3410587648, "nbytes": 8192}, {"name": "layers.6.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3410595840, "nbytes": 2097152}, {"name": "layers.6.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 3412692992, "nbytes": 1024}, {"name": "layers.6.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3412697088, "nbytes": 8388608}, {"name": "layers.6.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3421085696, "nbytes": 512}, {"name": "layers.6.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 3421089792, "nbytes": 8388608}, {"name": "layers.6.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 3429478400, "nbytes": 512}, {"name": "layers.6.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3429482496, "nbytes": 8388608}, {"name": "layers.6.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3437871104, "nbytes": 512}, {"name": "layers.6.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3437875200, "nbytes": 1572864}, {"name": "layers.6.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3439448064, "nbytes": 1572864}, {"name": "layers.6.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3441020928, "nbytes": 96}, {"name": "layers.6.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3441025024, "nbytes": 96}, {"name": "layers.6.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3441029120, "nbytes": 12}, {"name": "layers.6.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3441033216, "nbytes": 12}, {"name": "layers.7.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 3441037312, "nbytes": 4194304}, {"name": "layers.7.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 3445231616, "nbytes": 256}, {"name": "layers.7.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 3445235712, "nbytes": 2048}, {"name": "layers.7.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 3445239808, "nbytes": 33554432}, {"name": "layers.7.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 3478794240, "nbytes": 2048}, {"name": "layers.7.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 3478798336, "nbytes": 2097152}, {"name": "layers.7.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 3480895488, "nbytes": 128}, {"name": "layers.7.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3480899584, "nbytes": 1024}, {"name": "layers.7.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 3480903680, "nbytes": 67108864}, {"name": "layers.7.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 3548012544, "nbytes": 33554432}, {"name": "layers.7.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 3581566976, "nbytes": 2048}, {"name": "layers.7.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 3581571072, "nbytes": 256}, {"name": "layers.7.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 3581575168, "nbytes": 262144}, {"name": "layers.7.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 3581837312, "nbytes": 4194304}, {"name": "layers.7.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 3586031616, "nbytes": 4194304}, {"name": "layers.7.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3590225920, "nbytes": 1024}, {"name": "layers.7.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3590230016, "nbytes": 8192}, {"name": "layers.7.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3590238208, "nbytes": 8192}, {"name": "layers.7.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3590246400, "nbytes": 2097152}, {"name": "layers.7.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 3592343552, "nbytes": 1024}, {"name": "layers.7.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3592347648, "nbytes": 8388608}, {"name": "layers.7.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3600736256, "nbytes": 512}, {"name": "layers.7.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 3600740352, "nbytes": 8388608}, {"name": "layers.7.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 3609128960, "nbytes": 512}, {"name": "layers.7.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3609133056, "nbytes": 8388608}, {"name": "layers.7.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3617521664, "nbytes": 512}, {"name": "layers.7.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3617525760, "nbytes": 1572864}, {"name": "layers.7.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3619098624, "nbytes": 1572864}, {"name": "layers.7.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3620671488, "nbytes": 96}, {"name": "layers.7.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3620675584, "nbytes": 96}, {"name": "layers.7.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3620679680, "nbytes": 12}, {"name": "layers.7.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3620683776, "nbytes": 12}, {"name": "layers.8.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 3620687872, "nbytes": 4194304}, {"name": "layers.8.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 3624882176, "nbytes": 256}, {"name": "layers.8.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 3624886272, "nbytes": 2048}, {"name": "layers.8.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 3624890368, "nbytes": 33554432}, {"name": "layers.8.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 3658444800, "nbytes": 2048}, {"name": "layers.8.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 3658448896, "nbytes": 2097152}, {"name": "layers.8.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 3660546048, "nbytes": 128}, {"name": "layers.8.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3660550144, "nbytes": 1024}, {"name": "layers.8.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 3660554240, "nbytes": 67108864}, {"name": "layers.8.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 3727663104, "nbytes": 33554432}, {"name": "layers.8.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 3761217536, "nbytes": 2048}, {"name": "layers.8.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 3761221632, "nbytes": 256}, {"name": "layers.8.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 3761225728, "nbytes": 16384}, {"name": "layers.8.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 3761242112, "nbytes": 8388608}, {"name": "layers.8.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 3769630720, "nbytes": 8388608}, {"name": "layers.8.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3778019328, "nbytes": 1024}, {"name": "layers.8.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 3778023424, "nbytes": 8388608}, {"name": "layers.8.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 3786412032, "nbytes": 512}, {"name": "layers.8.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 3786416128, "nbytes": 524288}, {"name": "layers.8.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 3786940416, "nbytes": 4096}, {"name": "layers.8.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3786944512, "nbytes": 2097152}, {"name": "layers.8.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3789041664, "nbytes": 2097152}, {"name": "layers.8.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 3791138816, "nbytes": 256}, {"name": "layers.8.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3791142912, "nbytes": 8192}, {"name": "layers.8.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3791151104, "nbytes": 8192}, {"name": "layers.8.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3791159296, "nbytes": 2097152}, {"name": "layers.8.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 3793256448, "nbytes": 1024}, {"name": "layers.8.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3793260544, "nbytes": 8388608}, {"name": "layers.8.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3801649152, "nbytes": 512}, {"name": "layers.8.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 3801653248, "nbytes": 8388608}, {"name": "layers.8.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 3810041856, "nbytes": 512}, {"name": "layers.8.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3810045952, "nbytes": 8388608}, {"name": "layers.8.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3818434560, "nbytes": 512}, {"name": "layers.8.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3818438656, "nbytes": 1572864}, {"name": "layers.8.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3820011520, "nbytes": 1572864}, {"name": "layers.8.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3821584384, "nbytes": 96}, {"name": "layers.8.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 3821588480, "nbytes": 96}, {"name": "layers.8.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3821592576, "nbytes": 12}, {"name": "layers.8.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 3821596672, "nbytes": 12}, {"name": "layers.9.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 3821600768, "nbytes": 4194304}, {"name": "layers.9.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 3825795072, "nbytes": 256}, {"name": "layers.9.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 3825799168, "nbytes": 2048}, {"name": "layers.9.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 3825803264, "nbytes": 33554432}, {"name": "layers.9.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 3859357696, "nbytes": 2048}, {"name": "layers.9.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 3859361792, "nbytes": 2097152}, {"name": "layers.9.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 3861458944, "nbytes": 128}, {"name": "layers.9.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3861463040, "nbytes": 1024}, {"name": "layers.9.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 3861467136, "nbytes": 67108864}, {"name": "layers.9.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 3928576000, "nbytes": 33554432}, {"name": "layers.9.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 3962130432, "nbytes": 2048}, {"name": "layers.9.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 3962134528, "nbytes": 256}, {"name": "layers.9.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 3962138624, "nbytes": 262144}, {"name": "layers.9.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 3962400768, "nbytes": 4194304}, {"name": "layers.9.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 3966595072, "nbytes": 4194304}, {"name": "layers.9.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 3970789376, "nbytes": 1024}, {"name": "layers.9.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3970793472, "nbytes": 8192}, {"name": "layers.9.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 3970801664, "nbytes": 8192}, {"name": "layers.9.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 3970809856, "nbytes": 2097152}, {"name": "layers.9.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 3972907008, "nbytes": 1024}, {"name": "layers.9.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3972911104, "nbytes": 8388608}, {"name": "layers.9.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3981299712, "nbytes": 512}, {"name": "layers.9.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 3981303808, "nbytes": 8388608}, {"name": "layers.9.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 3989692416, "nbytes": 512}, {"name": "layers.9.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 3989696512, "nbytes": 8388608}, {"name": "layers.9.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 3998085120, "nbytes": 512}, {"name": "layers.9.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3998089216, "nbytes": 1572864}, {"name": "layers.9.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 3999662080, "nbytes": 1572864}, {"name": "layers.9.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4001234944, "nbytes": 96}, {"name": "layers.9.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4001239040, "nbytes": 96}, {"name": "layers.9.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4001243136, "nbytes": 12}, {"name": "layers.9.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4001247232, "nbytes": 12}, {"name": "layers.10.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 4001251328, "nbytes": 4194304}, {"name": "layers.10.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 4005445632, "nbytes": 256}, {"name": "layers.10.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 4005449728, "nbytes": 2048}, {"name": "layers.10.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 4005453824, "nbytes": 33554432}, {"name": "layers.10.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 4039008256, "nbytes": 2048}, {"name": "layers.10.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 4039012352, "nbytes": 2097152}, {"name": "layers.10.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 4041109504, "nbytes": 128}, {"name": "layers.10.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4041113600, "nbytes": 1024}, {"name": "layers.10.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 4041117696, "nbytes": 67108864}, {"name": "layers.10.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 4108226560, "nbytes": 33554432}, {"name": "layers.10.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 4141780992, "nbytes": 2048}, {"name": "layers.10.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 4141785088, "nbytes": 256}, {"name": "layers.10.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 4141789184, "nbytes": 16384}, {"name": "layers.10.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 4141805568, "nbytes": 8388608}, {"name": "layers.10.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 4150194176, "nbytes": 8388608}, {"name": "layers.10.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4158582784, "nbytes": 1024}, {"name": "layers.10.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 4158586880, "nbytes": 8388608}, {"name": "layers.10.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 4166975488, "nbytes": 512}, {"name": "layers.10.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 4166979584, "nbytes": 524288}, {"name": "layers.10.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 4167503872, "nbytes": 4096}, {"name": "layers.10.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4167507968, "nbytes": 2097152}, {"name": "layers.10.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4169605120, "nbytes": 2097152}, {"name": "layers.10.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4171702272, "nbytes": 256}, {"name": "layers.10.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4171706368, "nbytes": 8192}, {"name": "layers.10.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4171714560, "nbytes": 8192}, {"name": "layers.10.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4171722752, "nbytes": 2097152}, {"name": "layers.10.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 4173819904, "nbytes": 1024}, {"name": "layers.10.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4173824000, "nbytes": 8388608}, {"name": "layers.10.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4182212608, "nbytes": 512}, {"name": "layers.10.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 4182216704, "nbytes": 8388608}, {"name": "layers.10.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 4190605312, "nbytes": 512}, {"name": "layers.10.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4190609408, "nbytes": 8388608}, {"name": "layers.10.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4198998016, "nbytes": 512}, {"name": "layers.10.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4199002112, "nbytes": 1572864}, {"name": "layers.10.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4200574976, "nbytes": 1572864}, {"name": "layers.10.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4202147840, "nbytes": 96}, {"name": "layers.10.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4202151936, "nbytes": 96}, {"name": "layers.10.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4202156032, "nbytes": 12}, {"name": "layers.10.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4202160128, "nbytes": 12}, {"name": "layers.11.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 4202164224, "nbytes": 4194304}, {"name": "layers.11.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 4206358528, "nbytes": 256}, {"name": "layers.11.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 4206362624, "nbytes": 2048}, {"name": "layers.11.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 4206366720, "nbytes": 33554432}, {"name": "layers.11.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 4239921152, "nbytes": 2048}, {"name": "layers.11.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 4239925248, "nbytes": 2097152}, {"name": "layers.11.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 4242022400, "nbytes": 128}, {"name": "layers.11.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4242026496, "nbytes": 1024}, {"name": "layers.11.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 4242030592, "nbytes": 67108864}, {"name": "layers.11.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 4309139456, "nbytes": 33554432}, {"name": "layers.11.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 4342693888, "nbytes": 2048}, {"name": "layers.11.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 4342697984, "nbytes": 256}, {"name": "layers.11.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 4342702080, "nbytes": 262144}, {"name": "layers.11.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 4342964224, "nbytes": 4194304}, {"name": "layers.11.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 4347158528, "nbytes": 4194304}, {"name": "layers.11.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4351352832, "nbytes": 1024}, {"name": "layers.11.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4351356928, "nbytes": 8192}, {"name": "layers.11.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4351365120, "nbytes": 8192}, {"name": "layers.11.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4351373312, "nbytes": 2097152}, {"name": "layers.11.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 4353470464, "nbytes": 1024}, {"name": "layers.11.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4353474560, "nbytes": 8388608}, {"name": "layers.11.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4361863168, "nbytes": 512}, {"name": "layers.11.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 4361867264, "nbytes": 8388608}, {"name": "layers.11.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 4370255872, "nbytes": 512}, {"name": "layers.11.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4370259968, "nbytes": 8388608}, {"name": "layers.11.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4378648576, "nbytes": 512}, {"name": "layers.11.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4378652672, "nbytes": 1572864}, {"name": "layers.11.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4380225536, "nbytes": 1572864}, {"name": "layers.11.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4381798400, "nbytes": 96}, {"name": "layers.11.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4381802496, "nbytes": 96}, {"name": "layers.11.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4381806592, "nbytes": 12}, {"name": "layers.11.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4381810688, "nbytes": 12}, {"name": "layers.12.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 4381814784, "nbytes": 4194304}, {"name": "layers.12.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 4386009088, "nbytes": 256}, {"name": "layers.12.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 4386013184, "nbytes": 2048}, {"name": "layers.12.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 4386017280, "nbytes": 33554432}, {"name": "layers.12.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 4419571712, "nbytes": 2048}, {"name": "layers.12.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 4419575808, "nbytes": 2097152}, {"name": "layers.12.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 4421672960, "nbytes": 128}, {"name": "layers.12.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4421677056, "nbytes": 1024}, {"name": "layers.12.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 4421681152, "nbytes": 67108864}, {"name": "layers.12.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 4488790016, "nbytes": 33554432}, {"name": "layers.12.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 4522344448, "nbytes": 2048}, {"name": "layers.12.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 4522348544, "nbytes": 256}, {"name": "layers.12.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 4522352640, "nbytes": 16384}, {"name": "layers.12.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 4522369024, "nbytes": 8388608}, {"name": "layers.12.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 4530757632, "nbytes": 8388608}, {"name": "layers.12.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4539146240, "nbytes": 1024}, {"name": "layers.12.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 4539150336, "nbytes": 8388608}, {"name": "layers.12.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 4547538944, "nbytes": 512}, {"name": "layers.12.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 4547543040, "nbytes": 524288}, {"name": "layers.12.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 4548067328, "nbytes": 4096}, {"name": "layers.12.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4548071424, "nbytes": 2097152}, {"name": "layers.12.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4550168576, "nbytes": 2097152}, {"name": "layers.12.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4552265728, "nbytes": 256}, {"name": "layers.12.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4552269824, "nbytes": 8192}, {"name": "layers.12.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4552278016, "nbytes": 8192}, {"name": "layers.12.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4552286208, "nbytes": 2097152}, {"name": "layers.12.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 4554383360, "nbytes": 1024}, {"name": "layers.12.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4554387456, "nbytes": 8388608}, {"name": "layers.12.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4562776064, "nbytes": 512}, {"name": "layers.12.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 4562780160, "nbytes": 8388608}, {"name": "layers.12.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 4571168768, "nbytes": 512}, {"name": "layers.12.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4571172864, "nbytes": 8388608}, {"name": "layers.12.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4579561472, "nbytes": 512}, {"name": "layers.12.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4579565568, "nbytes": 1572864}, {"name": "layers.12.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4581138432, "nbytes": 1572864}, {"name": "layers.12.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4582711296, "nbytes": 96}, {"name": "layers.12.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4582715392, "nbytes": 96}, {"name": "layers.12.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4582719488, "nbytes": 12}, {"name": "layers.12.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4582723584, "nbytes": 12}, {"name": "layers.13.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 4582727680, "nbytes": 4194304}, {"name": "layers.13.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 4586921984, "nbytes": 256}, {"name": "layers.13.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 4586926080, "nbytes": 2048}, {"name": "layers.13.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 4586930176, "nbytes": 33554432}, {"name": "layers.13.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 4620484608, "nbytes": 2048}, {"name": "layers.13.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 4620488704, "nbytes": 2097152}, {"name": "layers.13.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 4622585856, "nbytes": 128}, {"name": "layers.13.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4622589952, "nbytes": 1024}, {"name": "layers.13.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 4622594048, "nbytes": 67108864}, {"name": "layers.13.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 4689702912, "nbytes": 33554432}, {"name": "layers.13.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 4723257344, "nbytes": 2048}, {"name": "layers.13.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 4723261440, "nbytes": 256}, {"name": "layers.13.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 4723265536, "nbytes": 262144}, {"name": "layers.13.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 4723527680, "nbytes": 4194304}, {"name": "layers.13.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 4727721984, "nbytes": 4194304}, {"name": "layers.13.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4731916288, "nbytes": 1024}, {"name": "layers.13.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4731920384, "nbytes": 8192}, {"name": "layers.13.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4731928576, "nbytes": 8192}, {"name": "layers.13.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4731936768, "nbytes": 2097152}, {"name": "layers.13.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 4734033920, "nbytes": 1024}, {"name": "layers.13.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4734038016, "nbytes": 8388608}, {"name": "layers.13.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4742426624, "nbytes": 512}, {"name": "layers.13.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 4742430720, "nbytes": 8388608}, {"name": "layers.13.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 4750819328, "nbytes": 512}, {"name": "layers.13.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4750823424, "nbytes": 8388608}, {"name": "layers.13.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4759212032, "nbytes": 512}, {"name": "layers.13.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4759216128, "nbytes": 1572864}, {"name": "layers.13.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4760788992, "nbytes": 1572864}, {"name": "layers.13.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4762361856, "nbytes": 96}, {"name": "layers.13.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4762365952, "nbytes": 96}, {"name": "layers.13.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4762370048, "nbytes": 12}, {"name": "layers.13.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4762374144, "nbytes": 12}, {"name": "layers.14.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 4762378240, "nbytes": 4194304}, {"name": "layers.14.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 4766572544, "nbytes": 256}, {"name": "layers.14.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 4766576640, "nbytes": 2048}, {"name": "layers.14.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 4766580736, "nbytes": 33554432}, {"name": "layers.14.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 4800135168, "nbytes": 2048}, {"name": "layers.14.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 4800139264, "nbytes": 2097152}, {"name": "layers.14.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 4802236416, "nbytes": 128}, {"name": "layers.14.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4802240512, "nbytes": 1024}, {"name": "layers.14.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 4802244608, "nbytes": 67108864}, {"name": "layers.14.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 4869353472, "nbytes": 33554432}, {"name": "layers.14.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 4902907904, "nbytes": 2048}, {"name": "layers.14.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 4902912000, "nbytes": 256}, {"name": "layers.14.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 4902916096, "nbytes": 16384}, {"name": "layers.14.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 4902932480, "nbytes": 8388608}, {"name": "layers.14.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 4911321088, "nbytes": 8388608}, {"name": "layers.14.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 4919709696, "nbytes": 1024}, {"name": "layers.14.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 4919713792, "nbytes": 8388608}, {"name": "layers.14.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 4928102400, "nbytes": 512}, {"name": "layers.14.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 4928106496, "nbytes": 524288}, {"name": "layers.14.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 4928630784, "nbytes": 4096}, {"name": "layers.14.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4928634880, "nbytes": 2097152}, {"name": "layers.14.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4930732032, "nbytes": 2097152}, {"name": "layers.14.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 4932829184, "nbytes": 256}, {"name": "layers.14.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4932833280, "nbytes": 8192}, {"name": "layers.14.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 4932841472, "nbytes": 8192}, {"name": "layers.14.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 4932849664, "nbytes": 2097152}, {"name": "layers.14.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 4934946816, "nbytes": 1024}, {"name": "layers.14.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4934950912, "nbytes": 8388608}, {"name": "layers.14.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4943339520, "nbytes": 512}, {"name": "layers.14.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 4943343616, "nbytes": 8388608}, {"name": "layers.14.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 4951732224, "nbytes": 512}, {"name": "layers.14.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 4951736320, "nbytes": 8388608}, {"name": "layers.14.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 4960124928, "nbytes": 512}, {"name": "layers.14.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4960129024, "nbytes": 1572864}, {"name": "layers.14.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 4961701888, "nbytes": 1572864}, {"name": "layers.14.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4963274752, "nbytes": 96}, {"name": "layers.14.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 4963278848, "nbytes": 96}, {"name": "layers.14.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4963282944, "nbytes": 12}, {"name": "layers.14.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 4963287040, "nbytes": 12}, {"name": "layers.15.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 4963291136, "nbytes": 4194304}, {"name": "layers.15.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 4967485440, "nbytes": 256}, {"name": "layers.15.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 4967489536, "nbytes": 2048}, {"name": "layers.15.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 4967493632, "nbytes": 33554432}, {"name": "layers.15.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 5001048064, "nbytes": 2048}, {"name": "layers.15.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 5001052160, "nbytes": 2097152}, {"name": "layers.15.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 5003149312, "nbytes": 128}, {"name": "layers.15.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5003153408, "nbytes": 1024}, {"name": "layers.15.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 5003157504, "nbytes": 67108864}, {"name": "layers.15.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 5070266368, "nbytes": 33554432}, {"name": "layers.15.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 5103820800, "nbytes": 2048}, {"name": "layers.15.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 5103824896, "nbytes": 256}, {"name": "layers.15.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 5103828992, "nbytes": 262144}, {"name": "layers.15.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 5104091136, "nbytes": 4194304}, {"name": "layers.15.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 5108285440, "nbytes": 4194304}, {"name": "layers.15.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5112479744, "nbytes": 1024}, {"name": "layers.15.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5112483840, "nbytes": 8192}, {"name": "layers.15.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5112492032, "nbytes": 8192}, {"name": "layers.15.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5112500224, "nbytes": 2097152}, {"name": "layers.15.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 5114597376, "nbytes": 1024}, {"name": "layers.15.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5114601472, "nbytes": 8388608}, {"name": "layers.15.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5122990080, "nbytes": 512}, {"name": "layers.15.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 5122994176, "nbytes": 8388608}, {"name": "layers.15.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 5131382784, "nbytes": 512}, {"name": "layers.15.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5131386880, "nbytes": 8388608}, {"name": "layers.15.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5139775488, "nbytes": 512}, {"name": "layers.15.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5139779584, "nbytes": 1572864}, {"name": "layers.15.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5141352448, "nbytes": 1572864}, {"name": "layers.15.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5142925312, "nbytes": 96}, {"name": "layers.15.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5142929408, "nbytes": 96}, {"name": "layers.15.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5142933504, "nbytes": 12}, {"name": "layers.15.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5142937600, "nbytes": 12}, {"name": "layers.16.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 5142941696, "nbytes": 4194304}, {"name": "layers.16.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 5147136000, "nbytes": 256}, {"name": "layers.16.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 5147140096, "nbytes": 2048}, {"name": "layers.16.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 5147144192, "nbytes": 33554432}, {"name": "layers.16.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 5180698624, "nbytes": 2048}, {"name": "layers.16.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 5180702720, "nbytes": 2097152}, {"name": "layers.16.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 5182799872, "nbytes": 128}, {"name": "layers.16.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5182803968, "nbytes": 1024}, {"name": "layers.16.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 5182808064, "nbytes": 67108864}, {"name": "layers.16.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 5249916928, "nbytes": 33554432}, {"name": "layers.16.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 5283471360, "nbytes": 2048}, {"name": "layers.16.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 5283475456, "nbytes": 256}, {"name": "layers.16.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 5283479552, "nbytes": 16384}, {"name": "layers.16.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 5283495936, "nbytes": 8388608}, {"name": "layers.16.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 5291884544, "nbytes": 8388608}, {"name": "layers.16.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5300273152, "nbytes": 1024}, {"name": "layers.16.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 5300277248, "nbytes": 8388608}, {"name": "layers.16.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 5308665856, "nbytes": 512}, {"name": "layers.16.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 5308669952, "nbytes": 524288}, {"name": "layers.16.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 5309194240, "nbytes": 4096}, {"name": "layers.16.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5309198336, "nbytes": 2097152}, {"name": "layers.16.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5311295488, "nbytes": 2097152}, {"name": "layers.16.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 5313392640, "nbytes": 256}, {"name": "layers.16.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5313396736, "nbytes": 8192}, {"name": "layers.16.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5313404928, "nbytes": 8192}, {"name": "layers.16.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5313413120, "nbytes": 2097152}, {"name": "layers.16.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 5315510272, "nbytes": 1024}, {"name": "layers.16.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5315514368, "nbytes": 8388608}, {"name": "layers.16.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5323902976, "nbytes": 512}, {"name": "layers.16.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 5323907072, "nbytes": 8388608}, {"name": "layers.16.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 5332295680, "nbytes": 512}, {"name": "layers.16.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5332299776, "nbytes": 8388608}, {"name": "layers.16.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5340688384, "nbytes": 512}, {"name": "layers.16.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5340692480, "nbytes": 1572864}, {"name": "layers.16.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5342265344, "nbytes": 1572864}, {"name": "layers.16.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5343838208, "nbytes": 96}, {"name": "layers.16.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5343842304, "nbytes": 96}, {"name": "layers.16.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5343846400, "nbytes": 12}, {"name": "layers.16.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5343850496, "nbytes": 12}, {"name": "layers.17.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 5343854592, "nbytes": 4194304}, {"name": "layers.17.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 5348048896, "nbytes": 256}, {"name": "layers.17.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 5348052992, "nbytes": 2048}, {"name": "layers.17.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 5348057088, "nbytes": 33554432}, {"name": "layers.17.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 5381611520, "nbytes": 2048}, {"name": "layers.17.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 5381615616, "nbytes": 2097152}, {"name": "layers.17.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 5383712768, "nbytes": 128}, {"name": "layers.17.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5383716864, "nbytes": 1024}, {"name": "layers.17.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 5383720960, "nbytes": 67108864}, {"name": "layers.17.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 5450829824, "nbytes": 33554432}, {"name": "layers.17.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 5484384256, "nbytes": 2048}, {"name": "layers.17.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 5484388352, "nbytes": 256}, {"name": "layers.17.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 5484392448, "nbytes": 262144}, {"name": "layers.17.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 5484654592, "nbytes": 4194304}, {"name": "layers.17.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 5488848896, "nbytes": 4194304}, {"name": "layers.17.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5493043200, "nbytes": 1024}, {"name": "layers.17.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5493047296, "nbytes": 8192}, {"name": "layers.17.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5493055488, "nbytes": 8192}, {"name": "layers.17.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5493063680, "nbytes": 2097152}, {"name": "layers.17.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 5495160832, "nbytes": 1024}, {"name": "layers.17.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5495164928, "nbytes": 8388608}, {"name": "layers.17.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5503553536, "nbytes": 512}, {"name": "layers.17.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 5503557632, "nbytes": 8388608}, {"name": "layers.17.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 5511946240, "nbytes": 512}, {"name": "layers.17.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5511950336, "nbytes": 8388608}, {"name": "layers.17.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5520338944, "nbytes": 512}, {"name": "layers.17.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5520343040, "nbytes": 1572864}, {"name": "layers.17.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5521915904, "nbytes": 1572864}, {"name": "layers.17.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5523488768, "nbytes": 96}, {"name": "layers.17.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5523492864, "nbytes": 96}, {"name": "layers.17.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5523496960, "nbytes": 12}, {"name": "layers.17.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5523501056, "nbytes": 12}, {"name": "layers.18.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 5523505152, "nbytes": 4194304}, {"name": "layers.18.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 5527699456, "nbytes": 256}, {"name": "layers.18.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 5527703552, "nbytes": 2048}, {"name": "layers.18.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 5527707648, "nbytes": 33554432}, {"name": "layers.18.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 5561262080, "nbytes": 2048}, {"name": "layers.18.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 5561266176, "nbytes": 2097152}, {"name": "layers.18.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 5563363328, "nbytes": 128}, {"name": "layers.18.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5563367424, "nbytes": 1024}, {"name": "layers.18.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 5563371520, "nbytes": 67108864}, {"name": "layers.18.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 5630480384, "nbytes": 33554432}, {"name": "layers.18.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 5664034816, "nbytes": 2048}, {"name": "layers.18.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 5664038912, "nbytes": 256}, {"name": "layers.18.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 5664043008, "nbytes": 16384}, {"name": "layers.18.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 5664059392, "nbytes": 8388608}, {"name": "layers.18.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 5672448000, "nbytes": 8388608}, {"name": "layers.18.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5680836608, "nbytes": 1024}, {"name": "layers.18.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 5680840704, "nbytes": 8388608}, {"name": "layers.18.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 5689229312, "nbytes": 512}, {"name": "layers.18.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 5689233408, "nbytes": 524288}, {"name": "layers.18.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 5689757696, "nbytes": 4096}, {"name": "layers.18.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5689761792, "nbytes": 2097152}, {"name": "layers.18.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5691858944, "nbytes": 2097152}, {"name": "layers.18.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 5693956096, "nbytes": 256}, {"name": "layers.18.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5693960192, "nbytes": 8192}, {"name": "layers.18.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5693968384, "nbytes": 8192}, {"name": "layers.18.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5693976576, "nbytes": 2097152}, {"name": "layers.18.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 5696073728, "nbytes": 1024}, {"name": "layers.18.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5696077824, "nbytes": 8388608}, {"name": "layers.18.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5704466432, "nbytes": 512}, {"name": "layers.18.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 5704470528, "nbytes": 8388608}, {"name": "layers.18.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 5712859136, "nbytes": 512}, {"name": "layers.18.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5712863232, "nbytes": 8388608}, {"name": "layers.18.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5721251840, "nbytes": 512}, {"name": "layers.18.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5721255936, "nbytes": 1572864}, {"name": "layers.18.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5722828800, "nbytes": 1572864}, {"name": "layers.18.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5724401664, "nbytes": 96}, {"name": "layers.18.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5724405760, "nbytes": 96}, {"name": "layers.18.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5724409856, "nbytes": 12}, {"name": "layers.18.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5724413952, "nbytes": 12}, {"name": "layers.19.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 5724418048, "nbytes": 4194304}, {"name": "layers.19.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 5728612352, "nbytes": 256}, {"name": "layers.19.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 5728616448, "nbytes": 2048}, {"name": "layers.19.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 5728620544, "nbytes": 33554432}, {"name": "layers.19.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 5762174976, "nbytes": 2048}, {"name": "layers.19.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 5762179072, "nbytes": 2097152}, {"name": "layers.19.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 5764276224, "nbytes": 128}, {"name": "layers.19.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5764280320, "nbytes": 1024}, {"name": "layers.19.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 5764284416, "nbytes": 67108864}, {"name": "layers.19.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 5831393280, "nbytes": 33554432}, {"name": "layers.19.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 5864947712, "nbytes": 2048}, {"name": "layers.19.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 5864951808, "nbytes": 256}, {"name": "layers.19.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 5864955904, "nbytes": 262144}, {"name": "layers.19.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 5865218048, "nbytes": 4194304}, {"name": "layers.19.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 5869412352, "nbytes": 4194304}, {"name": "layers.19.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5873606656, "nbytes": 1024}, {"name": "layers.19.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5873610752, "nbytes": 8192}, {"name": "layers.19.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 5873618944, "nbytes": 8192}, {"name": "layers.19.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 5873627136, "nbytes": 2097152}, {"name": "layers.19.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 5875724288, "nbytes": 1024}, {"name": "layers.19.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5875728384, "nbytes": 8388608}, {"name": "layers.19.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5884116992, "nbytes": 512}, {"name": "layers.19.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 5884121088, "nbytes": 8388608}, {"name": "layers.19.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 5892509696, "nbytes": 512}, {"name": "layers.19.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 5892513792, "nbytes": 8388608}, {"name": "layers.19.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 5900902400, "nbytes": 512}, {"name": "layers.19.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5900906496, "nbytes": 1572864}, {"name": "layers.19.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 5902479360, "nbytes": 1572864}, {"name": "layers.19.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5904052224, "nbytes": 96}, {"name": "layers.19.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 5904056320, "nbytes": 96}, {"name": "layers.19.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5904060416, "nbytes": 12}, {"name": "layers.19.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 5904064512, "nbytes": 12}, {"name": "layers.20.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 5904068608, "nbytes": 4194304}, {"name": "layers.20.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 5908262912, "nbytes": 256}, {"name": "layers.20.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 5908267008, "nbytes": 2048}, {"name": "layers.20.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 5908271104, "nbytes": 33554432}, {"name": "layers.20.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 5941825536, "nbytes": 2048}, {"name": "layers.20.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 5941829632, "nbytes": 2097152}, {"name": "layers.20.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 5943926784, "nbytes": 128}, {"name": "layers.20.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 5943930880, "nbytes": 1024}, {"name": "layers.20.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 5943934976, "nbytes": 67108864}, {"name": "layers.20.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 6011043840, "nbytes": 33554432}, {"name": "layers.20.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 6044598272, "nbytes": 2048}, {"name": "layers.20.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 6044602368, "nbytes": 256}, {"name": "layers.20.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 6044606464, "nbytes": 16384}, {"name": "layers.20.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 6044622848, "nbytes": 8388608}, {"name": "layers.20.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 6053011456, "nbytes": 8388608}, {"name": "layers.20.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6061400064, "nbytes": 1024}, {"name": "layers.20.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 6061404160, "nbytes": 8388608}, {"name": "layers.20.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 6069792768, "nbytes": 512}, {"name": "layers.20.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 6069796864, "nbytes": 524288}, {"name": "layers.20.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 6070321152, "nbytes": 4096}, {"name": "layers.20.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6070325248, "nbytes": 2097152}, {"name": "layers.20.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6072422400, "nbytes": 2097152}, {"name": "layers.20.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 6074519552, "nbytes": 256}, {"name": "layers.20.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6074523648, "nbytes": 8192}, {"name": "layers.20.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6074531840, "nbytes": 8192}, {"name": "layers.20.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6074540032, "nbytes": 2097152}, {"name": "layers.20.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 6076637184, "nbytes": 1024}, {"name": "layers.20.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6076641280, "nbytes": 8388608}, {"name": "layers.20.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6085029888, "nbytes": 512}, {"name": "layers.20.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 6085033984, "nbytes": 8388608}, {"name": "layers.20.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 6093422592, "nbytes": 512}, {"name": "layers.20.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6093426688, "nbytes": 8388608}, {"name": "layers.20.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6101815296, "nbytes": 512}, {"name": "layers.20.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6101819392, "nbytes": 1572864}, {"name": "layers.20.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6103392256, "nbytes": 1572864}, {"name": "layers.20.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6104965120, "nbytes": 96}, {"name": "layers.20.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6104969216, "nbytes": 96}, {"name": "layers.20.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6104973312, "nbytes": 12}, {"name": "layers.20.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6104977408, "nbytes": 12}, {"name": "layers.21.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 6104981504, "nbytes": 4194304}, {"name": "layers.21.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 6109175808, "nbytes": 256}, {"name": "layers.21.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 6109179904, "nbytes": 2048}, {"name": "layers.21.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 6109184000, "nbytes": 33554432}, {"name": "layers.21.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 6142738432, "nbytes": 2048}, {"name": "layers.21.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 6142742528, "nbytes": 2097152}, {"name": "layers.21.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 6144839680, "nbytes": 128}, {"name": "layers.21.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6144843776, "nbytes": 1024}, {"name": "layers.21.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 6144847872, "nbytes": 67108864}, {"name": "layers.21.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 6211956736, "nbytes": 33554432}, {"name": "layers.21.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 6245511168, "nbytes": 2048}, {"name": "layers.21.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 6245515264, "nbytes": 256}, {"name": "layers.21.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 6245519360, "nbytes": 262144}, {"name": "layers.21.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 6245781504, "nbytes": 4194304}, {"name": "layers.21.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 6249975808, "nbytes": 4194304}, {"name": "layers.21.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6254170112, "nbytes": 1024}, {"name": "layers.21.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6254174208, "nbytes": 8192}, {"name": "layers.21.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6254182400, "nbytes": 8192}, {"name": "layers.21.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6254190592, "nbytes": 2097152}, {"name": "layers.21.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 6256287744, "nbytes": 1024}, {"name": "layers.21.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6256291840, "nbytes": 8388608}, {"name": "layers.21.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6264680448, "nbytes": 512}, {"name": "layers.21.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 6264684544, "nbytes": 8388608}, {"name": "layers.21.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 6273073152, "nbytes": 512}, {"name": "layers.21.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6273077248, "nbytes": 8388608}, {"name": "layers.21.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6281465856, "nbytes": 512}, {"name": "layers.21.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6281469952, "nbytes": 1572864}, {"name": "layers.21.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6283042816, "nbytes": 1572864}, {"name": "layers.21.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6284615680, "nbytes": 96}, {"name": "layers.21.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6284619776, "nbytes": 96}, {"name": "layers.21.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6284623872, "nbytes": 12}, {"name": "layers.21.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6284627968, "nbytes": 12}, {"name": "layers.22.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 6284632064, "nbytes": 4194304}, {"name": "layers.22.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 6288826368, "nbytes": 256}, {"name": "layers.22.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 6288830464, "nbytes": 2048}, {"name": "layers.22.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 6288834560, "nbytes": 33554432}, {"name": "layers.22.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 6322388992, "nbytes": 2048}, {"name": "layers.22.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 6322393088, "nbytes": 2097152}, {"name": "layers.22.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 6324490240, "nbytes": 128}, {"name": "layers.22.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6324494336, "nbytes": 1024}, {"name": "layers.22.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 6324498432, "nbytes": 67108864}, {"name": "layers.22.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 6391607296, "nbytes": 33554432}, {"name": "layers.22.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 6425161728, "nbytes": 2048}, {"name": "layers.22.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 6425165824, "nbytes": 256}, {"name": "layers.22.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 6425169920, "nbytes": 16384}, {"name": "layers.22.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 6425186304, "nbytes": 8388608}, {"name": "layers.22.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 6433574912, "nbytes": 8388608}, {"name": "layers.22.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6441963520, "nbytes": 1024}, {"name": "layers.22.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 6441967616, "nbytes": 8388608}, {"name": "layers.22.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 6450356224, "nbytes": 512}, {"name": "layers.22.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 6450360320, "nbytes": 524288}, {"name": "layers.22.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 6450884608, "nbytes": 4096}, {"name": "layers.22.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6450888704, "nbytes": 2097152}, {"name": "layers.22.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6452985856, "nbytes": 2097152}, {"name": "layers.22.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 6455083008, "nbytes": 256}, {"name": "layers.22.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6455087104, "nbytes": 8192}, {"name": "layers.22.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6455095296, "nbytes": 8192}, {"name": "layers.22.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6455103488, "nbytes": 2097152}, {"name": "layers.22.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 6457200640, "nbytes": 1024}, {"name": "layers.22.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6457204736, "nbytes": 8388608}, {"name": "layers.22.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6465593344, "nbytes": 512}, {"name": "layers.22.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 6465597440, "nbytes": 8388608}, {"name": "layers.22.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 6473986048, "nbytes": 512}, {"name": "layers.22.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6473990144, "nbytes": 8388608}, {"name": "layers.22.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6482378752, "nbytes": 512}, {"name": "layers.22.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6482382848, "nbytes": 1572864}, {"name": "layers.22.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6483955712, "nbytes": 1572864}, {"name": "layers.22.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6485528576, "nbytes": 96}, {"name": "layers.22.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6485532672, "nbytes": 96}, {"name": "layers.22.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6485536768, "nbytes": 12}, {"name": "layers.22.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6485540864, "nbytes": 12}, {"name": "layers.23.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 6485544960, "nbytes": 4194304}, {"name": "layers.23.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 6489739264, "nbytes": 256}, {"name": "layers.23.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 6489743360, "nbytes": 2048}, {"name": "layers.23.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 6489747456, "nbytes": 33554432}, {"name": "layers.23.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 6523301888, "nbytes": 2048}, {"name": "layers.23.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 6523305984, "nbytes": 2097152}, {"name": "layers.23.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 6525403136, "nbytes": 128}, {"name": "layers.23.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6525407232, "nbytes": 1024}, {"name": "layers.23.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 6525411328, "nbytes": 67108864}, {"name": "layers.23.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 6592520192, "nbytes": 33554432}, {"name": "layers.23.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 6626074624, "nbytes": 2048}, {"name": "layers.23.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 6626078720, "nbytes": 256}, {"name": "layers.23.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 6626082816, "nbytes": 262144}, {"name": "layers.23.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 6626344960, "nbytes": 4194304}, {"name": "layers.23.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 6630539264, "nbytes": 4194304}, {"name": "layers.23.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6634733568, "nbytes": 1024}, {"name": "layers.23.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6634737664, "nbytes": 8192}, {"name": "layers.23.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6634745856, "nbytes": 8192}, {"name": "layers.23.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6634754048, "nbytes": 2097152}, {"name": "layers.23.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 6636851200, "nbytes": 1024}, {"name": "layers.23.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6636855296, "nbytes": 8388608}, {"name": "layers.23.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6645243904, "nbytes": 512}, {"name": "layers.23.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 6645248000, "nbytes": 8388608}, {"name": "layers.23.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 6653636608, "nbytes": 512}, {"name": "layers.23.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6653640704, "nbytes": 8388608}, {"name": "layers.23.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6662029312, "nbytes": 512}, {"name": "layers.23.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6662033408, "nbytes": 1572864}, {"name": "layers.23.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6663606272, "nbytes": 1572864}, {"name": "layers.23.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6665179136, "nbytes": 96}, {"name": "layers.23.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6665183232, "nbytes": 96}, {"name": "layers.23.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6665187328, "nbytes": 12}, {"name": "layers.23.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6665191424, "nbytes": 12}, {"name": "layers.24.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 6665195520, "nbytes": 4194304}, {"name": "layers.24.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 6669389824, "nbytes": 256}, {"name": "layers.24.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 6669393920, "nbytes": 2048}, {"name": "layers.24.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 6669398016, "nbytes": 33554432}, {"name": "layers.24.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 6702952448, "nbytes": 2048}, {"name": "layers.24.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 6702956544, "nbytes": 2097152}, {"name": "layers.24.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 6705053696, "nbytes": 128}, {"name": "layers.24.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6705057792, "nbytes": 1024}, {"name": "layers.24.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 6705061888, "nbytes": 67108864}, {"name": "layers.24.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 6772170752, "nbytes": 33554432}, {"name": "layers.24.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 6805725184, "nbytes": 2048}, {"name": "layers.24.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 6805729280, "nbytes": 256}, {"name": "layers.24.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 6805733376, "nbytes": 16384}, {"name": "layers.24.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 6805749760, "nbytes": 8388608}, {"name": "layers.24.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 6814138368, "nbytes": 8388608}, {"name": "layers.24.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6822526976, "nbytes": 1024}, {"name": "layers.24.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 6822531072, "nbytes": 8388608}, {"name": "layers.24.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 6830919680, "nbytes": 512}, {"name": "layers.24.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 6830923776, "nbytes": 524288}, {"name": "layers.24.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 6831448064, "nbytes": 4096}, {"name": "layers.24.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6831452160, "nbytes": 2097152}, {"name": "layers.24.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6833549312, "nbytes": 2097152}, {"name": "layers.24.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 6835646464, "nbytes": 256}, {"name": "layers.24.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6835650560, "nbytes": 8192}, {"name": "layers.24.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 6835658752, "nbytes": 8192}, {"name": "layers.24.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 6835666944, "nbytes": 2097152}, {"name": "layers.24.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 6837764096, "nbytes": 1024}, {"name": "layers.24.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6837768192, "nbytes": 8388608}, {"name": "layers.24.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6846156800, "nbytes": 512}, {"name": "layers.24.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 6846160896, "nbytes": 8388608}, {"name": "layers.24.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 6854549504, "nbytes": 512}, {"name": "layers.24.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 6854553600, "nbytes": 8388608}, {"name": "layers.24.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 6862942208, "nbytes": 512}, {"name": "layers.24.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6862946304, "nbytes": 1572864}, {"name": "layers.24.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 6864519168, "nbytes": 1572864}, {"name": "layers.24.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6866092032, "nbytes": 96}, {"name": "layers.24.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 6866096128, "nbytes": 96}, {"name": "layers.24.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6866100224, "nbytes": 12}, {"name": "layers.24.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 6866104320, "nbytes": 12}, {"name": "layers.25.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 6866108416, "nbytes": 4194304}, {"name": "layers.25.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 6870302720, "nbytes": 256}, {"name": "layers.25.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 6870306816, "nbytes": 2048}, {"name": "layers.25.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 6870310912, "nbytes": 33554432}, {"name": "layers.25.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 6903865344, "nbytes": 2048}, {"name": "layers.25.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 6903869440, "nbytes": 2097152}, {"name": "layers.25.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 6905966592, "nbytes": 128}, {"name": "layers.25.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 6905970688, "nbytes": 1024}, {"name": "layers.25.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 6905974784, "nbytes": 67108864}, {"name": "layers.25.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 6973083648, "nbytes": 33554432}, {"name": "layers.25.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 7006638080, "nbytes": 2048}, {"name": "layers.25.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 7006642176, "nbytes": 256}, {"name": "layers.25.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 7006646272, "nbytes": 262144}, {"name": "layers.25.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 7006908416, "nbytes": 4194304}, {"name": "layers.25.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 7011102720, "nbytes": 4194304}, {"name": "layers.25.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7015297024, "nbytes": 1024}, {"name": "layers.25.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7015301120, "nbytes": 8192}, {"name": "layers.25.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7015309312, "nbytes": 8192}, {"name": "layers.25.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7015317504, "nbytes": 2097152}, {"name": "layers.25.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 7017414656, "nbytes": 1024}, {"name": "layers.25.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7017418752, "nbytes": 8388608}, {"name": "layers.25.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7025807360, "nbytes": 512}, {"name": "layers.25.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 7025811456, "nbytes": 8388608}, {"name": "layers.25.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 7034200064, "nbytes": 512}, {"name": "layers.25.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7034204160, "nbytes": 8388608}, {"name": "layers.25.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7042592768, "nbytes": 512}, {"name": "layers.25.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7042596864, "nbytes": 1572864}, {"name": "layers.25.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7044169728, "nbytes": 1572864}, {"name": "layers.25.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7045742592, "nbytes": 96}, {"name": "layers.25.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7045746688, "nbytes": 96}, {"name": "layers.25.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7045750784, "nbytes": 12}, {"name": "layers.25.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7045754880, "nbytes": 12}, {"name": "layers.26.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 7045758976, "nbytes": 4194304}, {"name": "layers.26.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 7049953280, "nbytes": 256}, {"name": "layers.26.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 7049957376, "nbytes": 2048}, {"name": "layers.26.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 7049961472, "nbytes": 33554432}, {"name": "layers.26.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 7083515904, "nbytes": 2048}, {"name": "layers.26.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 7083520000, "nbytes": 2097152}, {"name": "layers.26.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 7085617152, "nbytes": 128}, {"name": "layers.26.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7085621248, "nbytes": 1024}, {"name": "layers.26.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 7085625344, "nbytes": 67108864}, {"name": "layers.26.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 7152734208, "nbytes": 33554432}, {"name": "layers.26.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 7186288640, "nbytes": 2048}, {"name": "layers.26.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 7186292736, "nbytes": 256}, {"name": "layers.26.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 7186296832, "nbytes": 16384}, {"name": "layers.26.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 7186313216, "nbytes": 8388608}, {"name": "layers.26.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 7194701824, "nbytes": 8388608}, {"name": "layers.26.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7203090432, "nbytes": 1024}, {"name": "layers.26.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 7203094528, "nbytes": 8388608}, {"name": "layers.26.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 7211483136, "nbytes": 512}, {"name": "layers.26.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 7211487232, "nbytes": 524288}, {"name": "layers.26.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 7212011520, "nbytes": 4096}, {"name": "layers.26.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7212015616, "nbytes": 2097152}, {"name": "layers.26.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7214112768, "nbytes": 2097152}, {"name": "layers.26.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 7216209920, "nbytes": 256}, {"name": "layers.26.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7216214016, "nbytes": 8192}, {"name": "layers.26.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7216222208, "nbytes": 8192}, {"name": "layers.26.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7216230400, "nbytes": 2097152}, {"name": "layers.26.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 7218327552, "nbytes": 1024}, {"name": "layers.26.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7218331648, "nbytes": 8388608}, {"name": "layers.26.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7226720256, "nbytes": 512}, {"name": "layers.26.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 7226724352, "nbytes": 8388608}, {"name": "layers.26.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 7235112960, "nbytes": 512}, {"name": "layers.26.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7235117056, "nbytes": 8388608}, {"name": "layers.26.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7243505664, "nbytes": 512}, {"name": "layers.26.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7243509760, "nbytes": 1572864}, {"name": "layers.26.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7245082624, "nbytes": 1572864}, {"name": "layers.26.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7246655488, "nbytes": 96}, {"name": "layers.26.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7246659584, "nbytes": 96}, {"name": "layers.26.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7246663680, "nbytes": 12}, {"name": "layers.26.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7246667776, "nbytes": 12}, {"name": "layers.27.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 7246671872, "nbytes": 4194304}, {"name": "layers.27.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 7250866176, "nbytes": 256}, {"name": "layers.27.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 7250870272, "nbytes": 2048}, {"name": "layers.27.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 7250874368, "nbytes": 33554432}, {"name": "layers.27.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 7284428800, "nbytes": 2048}, {"name": "layers.27.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 7284432896, "nbytes": 2097152}, {"name": "layers.27.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 7286530048, "nbytes": 128}, {"name": "layers.27.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7286534144, "nbytes": 1024}, {"name": "layers.27.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 7286538240, "nbytes": 67108864}, {"name": "layers.27.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 7353647104, "nbytes": 33554432}, {"name": "layers.27.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 7387201536, "nbytes": 2048}, {"name": "layers.27.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 7387205632, "nbytes": 256}, {"name": "layers.27.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 7387209728, "nbytes": 262144}, {"name": "layers.27.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 7387471872, "nbytes": 4194304}, {"name": "layers.27.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 7391666176, "nbytes": 4194304}, {"name": "layers.27.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7395860480, "nbytes": 1024}, {"name": "layers.27.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7395864576, "nbytes": 8192}, {"name": "layers.27.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7395872768, "nbytes": 8192}, {"name": "layers.27.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7395880960, "nbytes": 2097152}, {"name": "layers.27.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 7397978112, "nbytes": 1024}, {"name": "layers.27.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7397982208, "nbytes": 8388608}, {"name": "layers.27.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7406370816, "nbytes": 512}, {"name": "layers.27.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 7406374912, "nbytes": 8388608}, {"name": "layers.27.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 7414763520, "nbytes": 512}, {"name": "layers.27.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7414767616, "nbytes": 8388608}, {"name": "layers.27.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7423156224, "nbytes": 512}, {"name": "layers.27.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7423160320, "nbytes": 1572864}, {"name": "layers.27.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7424733184, "nbytes": 1572864}, {"name": "layers.27.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7426306048, "nbytes": 96}, {"name": "layers.27.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7426310144, "nbytes": 96}, {"name": "layers.27.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7426314240, "nbytes": 12}, {"name": "layers.27.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7426318336, "nbytes": 12}, {"name": "layers.28.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 7426322432, "nbytes": 4194304}, {"name": "layers.28.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 7430516736, "nbytes": 256}, {"name": "layers.28.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 7430520832, "nbytes": 2048}, {"name": "layers.28.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 7430524928, "nbytes": 33554432}, {"name": "layers.28.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 7464079360, "nbytes": 2048}, {"name": "layers.28.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 7464083456, "nbytes": 2097152}, {"name": "layers.28.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 7466180608, "nbytes": 128}, {"name": "layers.28.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7466184704, "nbytes": 1024}, {"name": "layers.28.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 7466188800, "nbytes": 67108864}, {"name": "layers.28.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 7533297664, "nbytes": 33554432}, {"name": "layers.28.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 7566852096, "nbytes": 2048}, {"name": "layers.28.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 7566856192, "nbytes": 256}, {"name": "layers.28.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 7566860288, "nbytes": 16384}, {"name": "layers.28.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 7566876672, "nbytes": 8388608}, {"name": "layers.28.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 7575265280, "nbytes": 8388608}, {"name": "layers.28.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7583653888, "nbytes": 1024}, {"name": "layers.28.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 7583657984, "nbytes": 8388608}, {"name": "layers.28.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 7592046592, "nbytes": 512}, {"name": "layers.28.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 7592050688, "nbytes": 524288}, {"name": "layers.28.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 7592574976, "nbytes": 4096}, {"name": "layers.28.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7592579072, "nbytes": 2097152}, {"name": "layers.28.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7594676224, "nbytes": 2097152}, {"name": "layers.28.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 7596773376, "nbytes": 256}, {"name": "layers.28.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7596777472, "nbytes": 8192}, {"name": "layers.28.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7596785664, "nbytes": 8192}, {"name": "layers.28.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7596793856, "nbytes": 2097152}, {"name": "layers.28.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 7598891008, "nbytes": 1024}, {"name": "layers.28.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7598895104, "nbytes": 8388608}, {"name": "layers.28.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7607283712, "nbytes": 512}, {"name": "layers.28.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 7607287808, "nbytes": 8388608}, {"name": "layers.28.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 7615676416, "nbytes": 512}, {"name": "layers.28.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7615680512, "nbytes": 8388608}, {"name": "layers.28.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7624069120, "nbytes": 512}, {"name": "layers.28.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7624073216, "nbytes": 1572864}, {"name": "layers.28.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7625646080, "nbytes": 1572864}, {"name": "layers.28.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7627218944, "nbytes": 96}, {"name": "layers.28.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7627223040, "nbytes": 96}, {"name": "layers.28.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7627227136, "nbytes": 12}, {"name": "layers.28.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7627231232, "nbytes": 12}, {"name": "layers.29.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 7627235328, "nbytes": 4194304}, {"name": "layers.29.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 7631429632, "nbytes": 256}, {"name": "layers.29.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 7631433728, "nbytes": 2048}, {"name": "layers.29.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 7631437824, "nbytes": 33554432}, {"name": "layers.29.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 7664992256, "nbytes": 2048}, {"name": "layers.29.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 7664996352, "nbytes": 2097152}, {"name": "layers.29.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 7667093504, "nbytes": 128}, {"name": "layers.29.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7667097600, "nbytes": 1024}, {"name": "layers.29.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 7667101696, "nbytes": 67108864}, {"name": "layers.29.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 7734210560, "nbytes": 33554432}, {"name": "layers.29.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 7767764992, "nbytes": 2048}, {"name": "layers.29.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 7767769088, "nbytes": 256}, {"name": "layers.29.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 7767773184, "nbytes": 262144}, {"name": "layers.29.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 7768035328, "nbytes": 4194304}, {"name": "layers.29.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 7772229632, "nbytes": 4194304}, {"name": "layers.29.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7776423936, "nbytes": 1024}, {"name": "layers.29.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7776428032, "nbytes": 8192}, {"name": "layers.29.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7776436224, "nbytes": 8192}, {"name": "layers.29.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7776444416, "nbytes": 2097152}, {"name": "layers.29.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 7778541568, "nbytes": 1024}, {"name": "layers.29.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7778545664, "nbytes": 8388608}, {"name": "layers.29.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7786934272, "nbytes": 512}, {"name": "layers.29.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 7786938368, "nbytes": 8388608}, {"name": "layers.29.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 7795326976, "nbytes": 512}, {"name": "layers.29.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7795331072, "nbytes": 8388608}, {"name": "layers.29.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7803719680, "nbytes": 512}, {"name": "layers.29.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7803723776, "nbytes": 1572864}, {"name": "layers.29.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 7805296640, "nbytes": 1572864}, {"name": "layers.29.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7806869504, "nbytes": 96}, {"name": "layers.29.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 7806873600, "nbytes": 96}, {"name": "layers.29.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7806877696, "nbytes": 12}, {"name": "layers.29.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 7806881792, "nbytes": 12}, {"name": "layers.30.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 7806885888, "nbytes": 4194304}, {"name": "layers.30.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 7811080192, "nbytes": 256}, {"name": "layers.30.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 7811084288, "nbytes": 2048}, {"name": "layers.30.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 7811088384, "nbytes": 33554432}, {"name": "layers.30.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 7844642816, "nbytes": 2048}, {"name": "layers.30.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 7844646912, "nbytes": 2097152}, {"name": "layers.30.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 7846744064, "nbytes": 128}, {"name": "layers.30.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7846748160, "nbytes": 1024}, {"name": "layers.30.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 7846752256, "nbytes": 67108864}, {"name": "layers.30.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 7913861120, "nbytes": 33554432}, {"name": "layers.30.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 7947415552, "nbytes": 2048}, {"name": "layers.30.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 7947419648, "nbytes": 256}, {"name": "layers.30.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 7947423744, "nbytes": 16384}, {"name": "layers.30.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 7947440128, "nbytes": 8388608}, {"name": "layers.30.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 7955828736, "nbytes": 8388608}, {"name": "layers.30.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 7964217344, "nbytes": 1024}, {"name": "layers.30.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 7964221440, "nbytes": 8388608}, {"name": "layers.30.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 7972610048, "nbytes": 512}, {"name": "layers.30.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 7972614144, "nbytes": 524288}, {"name": "layers.30.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 7973138432, "nbytes": 4096}, {"name": "layers.30.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7973142528, "nbytes": 2097152}, {"name": "layers.30.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7975239680, "nbytes": 2097152}, {"name": "layers.30.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 7977336832, "nbytes": 256}, {"name": "layers.30.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7977340928, "nbytes": 8192}, {"name": "layers.30.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 7977349120, "nbytes": 8192}, {"name": "layers.30.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 7977357312, "nbytes": 2097152}, {"name": "layers.30.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 7979454464, "nbytes": 1024}, {"name": "layers.30.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7979458560, "nbytes": 8388608}, {"name": "layers.30.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 7987847168, "nbytes": 512}, {"name": "layers.30.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 7987851264, "nbytes": 8388608}, {"name": "layers.30.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 7996239872, "nbytes": 512}, {"name": "layers.30.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 7996243968, "nbytes": 8388608}, {"name": "layers.30.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8004632576, "nbytes": 512}, {"name": "layers.30.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8004636672, "nbytes": 1572864}, {"name": "layers.30.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8006209536, "nbytes": 1572864}, {"name": "layers.30.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8007782400, "nbytes": 96}, {"name": "layers.30.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8007786496, "nbytes": 96}, {"name": "layers.30.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8007790592, "nbytes": 12}, {"name": "layers.30.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8007794688, "nbytes": 12}, {"name": "layers.31.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 8007798784, "nbytes": 4194304}, {"name": "layers.31.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 8011993088, "nbytes": 256}, {"name": "layers.31.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 8011997184, "nbytes": 2048}, {"name": "layers.31.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 8012001280, "nbytes": 33554432}, {"name": "layers.31.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 8045555712, "nbytes": 2048}, {"name": "layers.31.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 8045559808, "nbytes": 2097152}, {"name": "layers.31.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 8047656960, "nbytes": 128}, {"name": "layers.31.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8047661056, "nbytes": 1024}, {"name": "layers.31.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 8047665152, "nbytes": 67108864}, {"name": "layers.31.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 8114774016, "nbytes": 33554432}, {"name": "layers.31.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 8148328448, "nbytes": 2048}, {"name": "layers.31.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 8148332544, "nbytes": 256}, {"name": "layers.31.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 8148336640, "nbytes": 262144}, {"name": "layers.31.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 8148598784, "nbytes": 4194304}, {"name": "layers.31.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 8152793088, "nbytes": 4194304}, {"name": "layers.31.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8156987392, "nbytes": 1024}, {"name": "layers.31.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8156991488, "nbytes": 8192}, {"name": "layers.31.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8156999680, "nbytes": 8192}, {"name": "layers.31.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8157007872, "nbytes": 2097152}, {"name": "layers.31.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 8159105024, "nbytes": 1024}, {"name": "layers.31.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8159109120, "nbytes": 8388608}, {"name": "layers.31.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8167497728, "nbytes": 512}, {"name": "layers.31.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 8167501824, "nbytes": 8388608}, {"name": "layers.31.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 8175890432, "nbytes": 512}, {"name": "layers.31.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8175894528, "nbytes": 8388608}, {"name": "layers.31.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8184283136, "nbytes": 512}, {"name": "layers.31.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8184287232, "nbytes": 1572864}, {"name": "layers.31.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8185860096, "nbytes": 1572864}, {"name": "layers.31.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8187432960, "nbytes": 96}, {"name": "layers.31.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8187437056, "nbytes": 96}, {"name": "layers.31.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8187441152, "nbytes": 12}, {"name": "layers.31.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8187445248, "nbytes": 12}, {"name": "layers.32.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 8187449344, "nbytes": 4194304}, {"name": "layers.32.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 8191643648, "nbytes": 256}, {"name": "layers.32.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 8191647744, "nbytes": 2048}, {"name": "layers.32.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 8191651840, "nbytes": 33554432}, {"name": "layers.32.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 8225206272, "nbytes": 2048}, {"name": "layers.32.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 8225210368, "nbytes": 2097152}, {"name": "layers.32.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 8227307520, "nbytes": 128}, {"name": "layers.32.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8227311616, "nbytes": 1024}, {"name": "layers.32.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 8227315712, "nbytes": 67108864}, {"name": "layers.32.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 8294424576, "nbytes": 33554432}, {"name": "layers.32.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 8327979008, "nbytes": 2048}, {"name": "layers.32.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 8327983104, "nbytes": 256}, {"name": "layers.32.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 8327987200, "nbytes": 16384}, {"name": "layers.32.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 8328003584, "nbytes": 8388608}, {"name": "layers.32.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 8336392192, "nbytes": 8388608}, {"name": "layers.32.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8344780800, "nbytes": 1024}, {"name": "layers.32.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 8344784896, "nbytes": 8388608}, {"name": "layers.32.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 8353173504, "nbytes": 512}, {"name": "layers.32.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 8353177600, "nbytes": 524288}, {"name": "layers.32.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 8353701888, "nbytes": 4096}, {"name": "layers.32.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8353705984, "nbytes": 2097152}, {"name": "layers.32.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8355803136, "nbytes": 2097152}, {"name": "layers.32.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8357900288, "nbytes": 256}, {"name": "layers.32.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8357904384, "nbytes": 8192}, {"name": "layers.32.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8357912576, "nbytes": 8192}, {"name": "layers.32.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8357920768, "nbytes": 2097152}, {"name": "layers.32.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 8360017920, "nbytes": 1024}, {"name": "layers.32.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8360022016, "nbytes": 8388608}, {"name": "layers.32.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8368410624, "nbytes": 512}, {"name": "layers.32.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 8368414720, "nbytes": 8388608}, {"name": "layers.32.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 8376803328, "nbytes": 512}, {"name": "layers.32.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8376807424, "nbytes": 8388608}, {"name": "layers.32.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8385196032, "nbytes": 512}, {"name": "layers.32.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8385200128, "nbytes": 1572864}, {"name": "layers.32.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8386772992, "nbytes": 1572864}, {"name": "layers.32.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8388345856, "nbytes": 96}, {"name": "layers.32.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8388349952, "nbytes": 96}, {"name": "layers.32.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8388354048, "nbytes": 12}, {"name": "layers.32.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8388358144, "nbytes": 12}, {"name": "layers.33.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 8388362240, "nbytes": 4194304}, {"name": "layers.33.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 8392556544, "nbytes": 256}, {"name": "layers.33.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 8392560640, "nbytes": 2048}, {"name": "layers.33.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 8392564736, "nbytes": 33554432}, {"name": "layers.33.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 8426119168, "nbytes": 2048}, {"name": "layers.33.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 8426123264, "nbytes": 2097152}, {"name": "layers.33.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 8428220416, "nbytes": 128}, {"name": "layers.33.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8428224512, "nbytes": 1024}, {"name": "layers.33.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 8428228608, "nbytes": 67108864}, {"name": "layers.33.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 8495337472, "nbytes": 33554432}, {"name": "layers.33.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 8528891904, "nbytes": 2048}, {"name": "layers.33.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 8528896000, "nbytes": 256}, {"name": "layers.33.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 8528900096, "nbytes": 262144}, {"name": "layers.33.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 8529162240, "nbytes": 4194304}, {"name": "layers.33.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 8533356544, "nbytes": 4194304}, {"name": "layers.33.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8537550848, "nbytes": 1024}, {"name": "layers.33.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8537554944, "nbytes": 8192}, {"name": "layers.33.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8537563136, "nbytes": 8192}, {"name": "layers.33.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8537571328, "nbytes": 2097152}, {"name": "layers.33.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 8539668480, "nbytes": 1024}, {"name": "layers.33.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8539672576, "nbytes": 8388608}, {"name": "layers.33.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8548061184, "nbytes": 512}, {"name": "layers.33.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 8548065280, "nbytes": 8388608}, {"name": "layers.33.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 8556453888, "nbytes": 512}, {"name": "layers.33.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8556457984, "nbytes": 8388608}, {"name": "layers.33.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8564846592, "nbytes": 512}, {"name": "layers.33.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8564850688, "nbytes": 1572864}, {"name": "layers.33.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8566423552, "nbytes": 1572864}, {"name": "layers.33.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8567996416, "nbytes": 96}, {"name": "layers.33.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8568000512, "nbytes": 96}, {"name": "layers.33.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8568004608, "nbytes": 12}, {"name": "layers.33.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8568008704, "nbytes": 12}, {"name": "layers.34.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 8568012800, "nbytes": 4194304}, {"name": "layers.34.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 8572207104, "nbytes": 256}, {"name": "layers.34.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 8572211200, "nbytes": 2048}, {"name": "layers.34.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 8572215296, "nbytes": 33554432}, {"name": "layers.34.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 8605769728, "nbytes": 2048}, {"name": "layers.34.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 8605773824, "nbytes": 2097152}, {"name": "layers.34.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 8607870976, "nbytes": 128}, {"name": "layers.34.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8607875072, "nbytes": 1024}, {"name": "layers.34.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 8607879168, "nbytes": 67108864}, {"name": "layers.34.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 8674988032, "nbytes": 33554432}, {"name": "layers.34.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 8708542464, "nbytes": 2048}, {"name": "layers.34.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 8708546560, "nbytes": 256}, {"name": "layers.34.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 8708550656, "nbytes": 16384}, {"name": "layers.34.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 8708567040, "nbytes": 8388608}, {"name": "layers.34.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 8716955648, "nbytes": 8388608}, {"name": "layers.34.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8725344256, "nbytes": 1024}, {"name": "layers.34.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 8725348352, "nbytes": 8388608}, {"name": "layers.34.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 8733736960, "nbytes": 512}, {"name": "layers.34.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 8733741056, "nbytes": 524288}, {"name": "layers.34.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 8734265344, "nbytes": 4096}, {"name": "layers.34.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8734269440, "nbytes": 2097152}, {"name": "layers.34.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8736366592, "nbytes": 2097152}, {"name": "layers.34.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 8738463744, "nbytes": 256}, {"name": "layers.34.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8738467840, "nbytes": 8192}, {"name": "layers.34.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8738476032, "nbytes": 8192}, {"name": "layers.34.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8738484224, "nbytes": 2097152}, {"name": "layers.34.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 8740581376, "nbytes": 1024}, {"name": "layers.34.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8740585472, "nbytes": 8388608}, {"name": "layers.34.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8748974080, "nbytes": 512}, {"name": "layers.34.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 8748978176, "nbytes": 8388608}, {"name": "layers.34.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 8757366784, "nbytes": 512}, {"name": "layers.34.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8757370880, "nbytes": 8388608}, {"name": "layers.34.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8765759488, "nbytes": 512}, {"name": "layers.34.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8765763584, "nbytes": 1572864}, {"name": "layers.34.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8767336448, "nbytes": 1572864}, {"name": "layers.34.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8768909312, "nbytes": 96}, {"name": "layers.34.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8768913408, "nbytes": 96}, {"name": "layers.34.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8768917504, "nbytes": 12}, {"name": "layers.34.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8768921600, "nbytes": 12}, {"name": "layers.35.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 8768925696, "nbytes": 4194304}, {"name": "layers.35.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 8773120000, "nbytes": 256}, {"name": "layers.35.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 8773124096, "nbytes": 2048}, {"name": "layers.35.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 8773128192, "nbytes": 33554432}, {"name": "layers.35.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 8806682624, "nbytes": 2048}, {"name": "layers.35.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 8806686720, "nbytes": 2097152}, {"name": "layers.35.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 8808783872, "nbytes": 128}, {"name": "layers.35.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8808787968, "nbytes": 1024}, {"name": "layers.35.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 8808792064, "nbytes": 67108864}, {"name": "layers.35.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 8875900928, "nbytes": 33554432}, {"name": "layers.35.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 8909455360, "nbytes": 2048}, {"name": "layers.35.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 8909459456, "nbytes": 256}, {"name": "layers.35.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 8909463552, "nbytes": 262144}, {"name": "layers.35.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 8909725696, "nbytes": 4194304}, {"name": "layers.35.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 8913920000, "nbytes": 4194304}, {"name": "layers.35.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8918114304, "nbytes": 1024}, {"name": "layers.35.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8918118400, "nbytes": 8192}, {"name": "layers.35.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 8918126592, "nbytes": 8192}, {"name": "layers.35.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 8918134784, "nbytes": 2097152}, {"name": "layers.35.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 8920231936, "nbytes": 1024}, {"name": "layers.35.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8920236032, "nbytes": 8388608}, {"name": "layers.35.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8928624640, "nbytes": 512}, {"name": "layers.35.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 8928628736, "nbytes": 8388608}, {"name": "layers.35.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 8937017344, "nbytes": 512}, {"name": "layers.35.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 8937021440, "nbytes": 8388608}, {"name": "layers.35.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 8945410048, "nbytes": 512}, {"name": "layers.35.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8945414144, "nbytes": 1572864}, {"name": "layers.35.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 8946987008, "nbytes": 1572864}, {"name": "layers.35.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8948559872, "nbytes": 96}, {"name": "layers.35.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 8948563968, "nbytes": 96}, {"name": "layers.35.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8948568064, "nbytes": 12}, {"name": "layers.35.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 8948572160, "nbytes": 12}, {"name": "layers.36.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 8948576256, "nbytes": 4194304}, {"name": "layers.36.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 8952770560, "nbytes": 256}, {"name": "layers.36.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 8952774656, "nbytes": 2048}, {"name": "layers.36.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 8952778752, "nbytes": 33554432}, {"name": "layers.36.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 8986333184, "nbytes": 2048}, {"name": "layers.36.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 8986337280, "nbytes": 2097152}, {"name": "layers.36.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 8988434432, "nbytes": 128}, {"name": "layers.36.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 8988438528, "nbytes": 1024}, {"name": "layers.36.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 8988442624, "nbytes": 67108864}, {"name": "layers.36.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 9055551488, "nbytes": 33554432}, {"name": "layers.36.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 9089105920, "nbytes": 2048}, {"name": "layers.36.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 9089110016, "nbytes": 256}, {"name": "layers.36.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 9089114112, "nbytes": 16384}, {"name": "layers.36.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 9089130496, "nbytes": 8388608}, {"name": "layers.36.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 9097519104, "nbytes": 8388608}, {"name": "layers.36.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9105907712, "nbytes": 1024}, {"name": "layers.36.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 9105911808, "nbytes": 8388608}, {"name": "layers.36.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 9114300416, "nbytes": 512}, {"name": "layers.36.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 9114304512, "nbytes": 524288}, {"name": "layers.36.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 9114828800, "nbytes": 4096}, {"name": "layers.36.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9114832896, "nbytes": 2097152}, {"name": "layers.36.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9116930048, "nbytes": 2097152}, {"name": "layers.36.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9119027200, "nbytes": 256}, {"name": "layers.36.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9119031296, "nbytes": 8192}, {"name": "layers.36.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9119039488, "nbytes": 8192}, {"name": "layers.36.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9119047680, "nbytes": 2097152}, {"name": "layers.36.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 9121144832, "nbytes": 1024}, {"name": "layers.36.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9121148928, "nbytes": 8388608}, {"name": "layers.36.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9129537536, "nbytes": 512}, {"name": "layers.36.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 9129541632, "nbytes": 8388608}, {"name": "layers.36.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 9137930240, "nbytes": 512}, {"name": "layers.36.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9137934336, "nbytes": 8388608}, {"name": "layers.36.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9146322944, "nbytes": 512}, {"name": "layers.36.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9146327040, "nbytes": 1572864}, {"name": "layers.36.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9147899904, "nbytes": 1572864}, {"name": "layers.36.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9149472768, "nbytes": 96}, {"name": "layers.36.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9149476864, "nbytes": 96}, {"name": "layers.36.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9149480960, "nbytes": 12}, {"name": "layers.36.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9149485056, "nbytes": 12}, {"name": "layers.37.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 9149489152, "nbytes": 4194304}, {"name": "layers.37.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 9153683456, "nbytes": 256}, {"name": "layers.37.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 9153687552, "nbytes": 2048}, {"name": "layers.37.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 9153691648, "nbytes": 33554432}, {"name": "layers.37.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 9187246080, "nbytes": 2048}, {"name": "layers.37.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 9187250176, "nbytes": 2097152}, {"name": "layers.37.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 9189347328, "nbytes": 128}, {"name": "layers.37.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9189351424, "nbytes": 1024}, {"name": "layers.37.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 9189355520, "nbytes": 67108864}, {"name": "layers.37.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 9256464384, "nbytes": 33554432}, {"name": "layers.37.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 9290018816, "nbytes": 2048}, {"name": "layers.37.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 9290022912, "nbytes": 256}, {"name": "layers.37.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 9290027008, "nbytes": 262144}, {"name": "layers.37.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 9290289152, "nbytes": 4194304}, {"name": "layers.37.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 9294483456, "nbytes": 4194304}, {"name": "layers.37.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9298677760, "nbytes": 1024}, {"name": "layers.37.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9298681856, "nbytes": 8192}, {"name": "layers.37.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9298690048, "nbytes": 8192}, {"name": "layers.37.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9298698240, "nbytes": 2097152}, {"name": "layers.37.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 9300795392, "nbytes": 1024}, {"name": "layers.37.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9300799488, "nbytes": 8388608}, {"name": "layers.37.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9309188096, "nbytes": 512}, {"name": "layers.37.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 9309192192, "nbytes": 8388608}, {"name": "layers.37.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 9317580800, "nbytes": 512}, {"name": "layers.37.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9317584896, "nbytes": 8388608}, {"name": "layers.37.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9325973504, "nbytes": 512}, {"name": "layers.37.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9325977600, "nbytes": 1572864}, {"name": "layers.37.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9327550464, "nbytes": 1572864}, {"name": "layers.37.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9329123328, "nbytes": 96}, {"name": "layers.37.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9329127424, "nbytes": 96}, {"name": "layers.37.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9329131520, "nbytes": 12}, {"name": "layers.37.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9329135616, "nbytes": 12}, {"name": "layers.38.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 9329139712, "nbytes": 4194304}, {"name": "layers.38.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 9333334016, "nbytes": 256}, {"name": "layers.38.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 9333338112, "nbytes": 2048}, {"name": "layers.38.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 9333342208, "nbytes": 33554432}, {"name": "layers.38.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 9366896640, "nbytes": 2048}, {"name": "layers.38.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 9366900736, "nbytes": 2097152}, {"name": "layers.38.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 9368997888, "nbytes": 128}, {"name": "layers.38.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9369001984, "nbytes": 1024}, {"name": "layers.38.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 9369006080, "nbytes": 67108864}, {"name": "layers.38.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 9436114944, "nbytes": 33554432}, {"name": "layers.38.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 9469669376, "nbytes": 2048}, {"name": "layers.38.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 9469673472, "nbytes": 256}, {"name": "layers.38.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 9469677568, "nbytes": 16384}, {"name": "layers.38.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 9469693952, "nbytes": 8388608}, {"name": "layers.38.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 9478082560, "nbytes": 8388608}, {"name": "layers.38.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9486471168, "nbytes": 1024}, {"name": "layers.38.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 9486475264, "nbytes": 8388608}, {"name": "layers.38.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 9494863872, "nbytes": 512}, {"name": "layers.38.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 9494867968, "nbytes": 524288}, {"name": "layers.38.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 9495392256, "nbytes": 4096}, {"name": "layers.38.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9495396352, "nbytes": 2097152}, {"name": "layers.38.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9497493504, "nbytes": 2097152}, {"name": "layers.38.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9499590656, "nbytes": 256}, {"name": "layers.38.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9499594752, "nbytes": 8192}, {"name": "layers.38.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9499602944, "nbytes": 8192}, {"name": "layers.38.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9499611136, "nbytes": 2097152}, {"name": "layers.38.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 9501708288, "nbytes": 1024}, {"name": "layers.38.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9501712384, "nbytes": 8388608}, {"name": "layers.38.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9510100992, "nbytes": 512}, {"name": "layers.38.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 9510105088, "nbytes": 8388608}, {"name": "layers.38.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 9518493696, "nbytes": 512}, {"name": "layers.38.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9518497792, "nbytes": 8388608}, {"name": "layers.38.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9526886400, "nbytes": 512}, {"name": "layers.38.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9526890496, "nbytes": 1572864}, {"name": "layers.38.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9528463360, "nbytes": 1572864}, {"name": "layers.38.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9530036224, "nbytes": 96}, {"name": "layers.38.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9530040320, "nbytes": 96}, {"name": "layers.38.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9530044416, "nbytes": 12}, {"name": "layers.38.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9530048512, "nbytes": 12}, {"name": "layers.39.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 9530052608, "nbytes": 4194304}, {"name": "layers.39.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 9534246912, "nbytes": 256}, {"name": "layers.39.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 9534251008, "nbytes": 2048}, {"name": "layers.39.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 9534255104, "nbytes": 33554432}, {"name": "layers.39.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 9567809536, "nbytes": 2048}, {"name": "layers.39.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 9567813632, "nbytes": 2097152}, {"name": "layers.39.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 9569910784, "nbytes": 128}, {"name": "layers.39.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9569914880, "nbytes": 1024}, {"name": "layers.39.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 9569918976, "nbytes": 67108864}, {"name": "layers.39.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 9637027840, "nbytes": 33554432}, {"name": "layers.39.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 9670582272, "nbytes": 2048}, {"name": "layers.39.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 9670586368, "nbytes": 256}, {"name": "layers.39.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 9670590464, "nbytes": 262144}, {"name": "layers.39.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 9670852608, "nbytes": 4194304}, {"name": "layers.39.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 9675046912, "nbytes": 4194304}, {"name": "layers.39.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9679241216, "nbytes": 1024}, {"name": "layers.39.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9679245312, "nbytes": 8192}, {"name": "layers.39.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9679253504, "nbytes": 8192}, {"name": "layers.39.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9679261696, "nbytes": 2097152}, {"name": "layers.39.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 9681358848, "nbytes": 1024}, {"name": "layers.39.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9681362944, "nbytes": 8388608}, {"name": "layers.39.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9689751552, "nbytes": 512}, {"name": "layers.39.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 9689755648, "nbytes": 8388608}, {"name": "layers.39.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 9698144256, "nbytes": 512}, {"name": "layers.39.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9698148352, "nbytes": 8388608}, {"name": "layers.39.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9706536960, "nbytes": 512}, {"name": "layers.39.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9706541056, "nbytes": 1572864}, {"name": "layers.39.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9708113920, "nbytes": 1572864}, {"name": "layers.39.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9709686784, "nbytes": 96}, {"name": "layers.39.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9709690880, "nbytes": 96}, {"name": "layers.39.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9709694976, "nbytes": 12}, {"name": "layers.39.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9709699072, "nbytes": 12}, {"name": "layers.40.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 9709703168, "nbytes": 4194304}, {"name": "layers.40.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 9713897472, "nbytes": 256}, {"name": "layers.40.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 9713901568, "nbytes": 2048}, {"name": "layers.40.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 9713905664, "nbytes": 33554432}, {"name": "layers.40.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 9747460096, "nbytes": 2048}, {"name": "layers.40.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 9747464192, "nbytes": 2097152}, {"name": "layers.40.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 9749561344, "nbytes": 128}, {"name": "layers.40.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9749565440, "nbytes": 1024}, {"name": "layers.40.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 9749569536, "nbytes": 67108864}, {"name": "layers.40.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 9816678400, "nbytes": 33554432}, {"name": "layers.40.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 9850232832, "nbytes": 2048}, {"name": "layers.40.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 9850236928, "nbytes": 256}, {"name": "layers.40.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 9850241024, "nbytes": 16384}, {"name": "layers.40.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 9850257408, "nbytes": 8388608}, {"name": "layers.40.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 9858646016, "nbytes": 8388608}, {"name": "layers.40.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9867034624, "nbytes": 1024}, {"name": "layers.40.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 9867038720, "nbytes": 8388608}, {"name": "layers.40.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 9875427328, "nbytes": 512}, {"name": "layers.40.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 9875431424, "nbytes": 524288}, {"name": "layers.40.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 9875955712, "nbytes": 4096}, {"name": "layers.40.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9875959808, "nbytes": 2097152}, {"name": "layers.40.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9878056960, "nbytes": 2097152}, {"name": "layers.40.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 9880154112, "nbytes": 256}, {"name": "layers.40.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9880158208, "nbytes": 8192}, {"name": "layers.40.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 9880166400, "nbytes": 8192}, {"name": "layers.40.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 9880174592, "nbytes": 2097152}, {"name": "layers.40.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 9882271744, "nbytes": 1024}, {"name": "layers.40.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9882275840, "nbytes": 8388608}, {"name": "layers.40.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9890664448, "nbytes": 512}, {"name": "layers.40.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 9890668544, "nbytes": 8388608}, {"name": "layers.40.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 9899057152, "nbytes": 512}, {"name": "layers.40.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 9899061248, "nbytes": 8388608}, {"name": "layers.40.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 9907449856, "nbytes": 512}, {"name": "layers.40.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9907453952, "nbytes": 1572864}, {"name": "layers.40.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 9909026816, "nbytes": 1572864}, {"name": "layers.40.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9910599680, "nbytes": 96}, {"name": "layers.40.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 9910603776, "nbytes": 96}, {"name": "layers.40.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9910607872, "nbytes": 12}, {"name": "layers.40.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 9910611968, "nbytes": 12}, {"name": "layers.41.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 9910616064, "nbytes": 4194304}, {"name": "layers.41.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 9914810368, "nbytes": 256}, {"name": "layers.41.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 9914814464, "nbytes": 2048}, {"name": "layers.41.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 9914818560, "nbytes": 33554432}, {"name": "layers.41.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 9948372992, "nbytes": 2048}, {"name": "layers.41.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 9948377088, "nbytes": 2097152}, {"name": "layers.41.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 9950474240, "nbytes": 128}, {"name": "layers.41.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 9950478336, "nbytes": 1024}, {"name": "layers.41.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 9950482432, "nbytes": 67108864}, {"name": "layers.41.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 10017591296, "nbytes": 33554432}, {"name": "layers.41.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 10051145728, "nbytes": 2048}, {"name": "layers.41.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 10051149824, "nbytes": 256}, {"name": "layers.41.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [128, 512], "global_off": 10051153920, "nbytes": 262144}, {"name": "layers.41.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 10051416064, "nbytes": 4194304}, {"name": "layers.41.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512, 4096], "global_off": 10055610368, "nbytes": 4194304}, {"name": "layers.41.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 10059804672, "nbytes": 1024}, {"name": "layers.41.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 10059808768, "nbytes": 8192}, {"name": "layers.41.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 10059816960, "nbytes": 8192}, {"name": "layers.41.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 10059825152, "nbytes": 2097152}, {"name": "layers.41.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 10061922304, "nbytes": 1024}, {"name": "layers.41.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 10061926400, "nbytes": 8388608}, {"name": "layers.41.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 10070315008, "nbytes": 512}, {"name": "layers.41.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 10070319104, "nbytes": 8388608}, {"name": "layers.41.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 10078707712, "nbytes": 512}, {"name": "layers.41.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 10078711808, "nbytes": 8388608}, {"name": "layers.41.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 10087100416, "nbytes": 512}, {"name": "layers.41.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 10087104512, "nbytes": 1572864}, {"name": "layers.41.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 10088677376, "nbytes": 1572864}, {"name": "layers.41.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 10090250240, "nbytes": 96}, {"name": "layers.41.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 10090254336, "nbytes": 96}, {"name": "layers.41.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 10090258432, "nbytes": 12}, {"name": "layers.41.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 10090262528, "nbytes": 12}, {"name": "layers.42.attn.wq_a.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [1024, 4096], "global_off": 10090266624, "nbytes": 4194304}, {"name": "layers.42.attn.wq_a.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [8, 32], "global_off": 10094460928, "nbytes": 256}, {"name": "layers.42.attn.q_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024], "global_off": 10094465024, "nbytes": 2048}, {"name": "layers.42.attn.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [32768, 1024], "global_off": 10094469120, "nbytes": 33554432}, {"name": "layers.42.attn.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [256, 8], "global_off": 10128023552, "nbytes": 2048}, {"name": "layers.42.attn.wkv.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [512, 4096], "global_off": 10128027648, "nbytes": 2097152}, {"name": "layers.42.attn.wkv.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [4, 32], "global_off": 10130124800, "nbytes": 128}, {"name": "layers.42.attn.kv_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 10130128896, "nbytes": 1024}, {"name": "layers.42.attn.wo_a", "kind": "weight", "dtype": "bfloat16", "shape": [8192, 4096], "global_off": 10130132992, "nbytes": 67108864}, {"name": "layers.42.attn.wo_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 8192], "global_off": 10197241856, "nbytes": 33554432}, {"name": "layers.42.attn.wo_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 64], "global_off": 10230796288, "nbytes": 2048}, {"name": "layers.42.attn.attn_sink", "kind": "weight", "dtype": "float32", "shape": [64], "global_off": 10230800384, "nbytes": 256}, {"name": "layers.42.attn.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 1024], "global_off": 10230804480, "nbytes": 16384}, {"name": "layers.42.attn.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 10230820864, "nbytes": 8388608}, {"name": "layers.42.attn.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [1024, 4096], "global_off": 10239209472, "nbytes": 8388608}, {"name": "layers.42.attn.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [512], "global_off": 10247598080, "nbytes": 1024}, {"name": "layers.42.attn.indexer.wq_b.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [8192, 1024], "global_off": 10247602176, "nbytes": 8388608}, {"name": "layers.42.attn.indexer.wq_b.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [64, 8], "global_off": 10255990784, "nbytes": 512}, {"name": "layers.42.attn.indexer.weights_proj.weight", "kind": "weight", "dtype": "bfloat16", "shape": [64, 4096], "global_off": 10255994880, "nbytes": 524288}, {"name": "layers.42.attn.indexer.compressor.ape", "kind": "weight", "dtype": "float32", "shape": [4, 256], "global_off": 10256519168, "nbytes": 4096}, {"name": "layers.42.attn.indexer.compressor.wkv.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 10256523264, "nbytes": 2097152}, {"name": "layers.42.attn.indexer.compressor.wgate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 10258620416, "nbytes": 2097152}, {"name": "layers.42.attn.indexer.compressor.norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [128], "global_off": 10260717568, "nbytes": 256}, {"name": "layers.42.attn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 10260721664, "nbytes": 8192}, {"name": "layers.42.ffn_norm.weight", "kind": "weight", "dtype": "bfloat16", "shape": [4096], "global_off": 10260729856, "nbytes": 8192}, {"name": "layers.42.ffn.gate.weight", "kind": "weight", "dtype": "bfloat16", "shape": [256, 4096], "global_off": 10260738048, "nbytes": 2097152}, {"name": "layers.42.ffn.gate.bias", "kind": "weight", "dtype": "float32", "shape": [256], "global_off": 10262835200, "nbytes": 1024}, {"name": "layers.42.ffn.shared_experts.w1.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 10262839296, "nbytes": 8388608}, {"name": "layers.42.ffn.shared_experts.w1.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 10271227904, "nbytes": 512}, {"name": "layers.42.ffn.shared_experts.w2.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [4096, 2048], "global_off": 10271232000, "nbytes": 8388608}, {"name": "layers.42.ffn.shared_experts.w2.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [32, 16], "global_off": 10279620608, "nbytes": 512}, {"name": "layers.42.ffn.shared_experts.w3.weight", "kind": "weight", "dtype": "float8_e4m3fn", "shape": [2048, 4096], "global_off": 10279624704, "nbytes": 8388608}, {"name": "layers.42.ffn.shared_experts.w3.scale", "kind": "weight", "dtype": "float8_e8m0fnu", "shape": [16, 32], "global_off": 10288013312, "nbytes": 512}, {"name": "layers.42.hc_attn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 10288017408, "nbytes": 1572864}, {"name": "layers.42.hc_ffn_fn", "kind": "weight", "dtype": "float32", "shape": [24, 16384], "global_off": 10289590272, "nbytes": 1572864}, {"name": "layers.42.hc_attn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 10291163136, "nbytes": 96}, {"name": "layers.42.hc_ffn_base", "kind": "weight", "dtype": "float32", "shape": [24], "global_off": 10291167232, "nbytes": 96}, {"name": "layers.42.hc_attn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 10291171328, "nbytes": 12}, {"name": "layers.42.hc_ffn_scale", "kind": "weight", "dtype": "float32", "shape": [3], "global_off": 10291175424, "nbytes": 12}, {"name": "gate_up_packed#L00000", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 10291179520, "nbytes": 2147483648}, {"name": "gate_up_scale#L00000", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 12438663168, "nbytes": 134217728}, {"name": "down_packed#L00000", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 12572880896, "nbytes": 1073741824}, {"name": "down_scale#L00000", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 13646622720, "nbytes": 67108864}, {"name": "gate_up_packed#L00001", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 13713731584, "nbytes": 2147483648}, {"name": "gate_up_scale#L00001", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 15861215232, "nbytes": 134217728}, {"name": "down_packed#L00001", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 15995432960, "nbytes": 1073741824}, {"name": "down_scale#L00001", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 17069174784, "nbytes": 67108864}, {"name": "gate_up_packed#L00002", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 17136283648, "nbytes": 2147483648}, {"name": "gate_up_scale#L00002", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 19283767296, "nbytes": 134217728}, {"name": "down_packed#L00002", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 19417985024, "nbytes": 1073741824}, {"name": "down_scale#L00002", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 20491726848, "nbytes": 67108864}, {"name": "gate_up_packed#L00003", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 20558835712, "nbytes": 2147483648}, {"name": "gate_up_scale#L00003", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 22706319360, "nbytes": 134217728}, {"name": "down_packed#L00003", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 22840537088, "nbytes": 1073741824}, {"name": "down_scale#L00003", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 23914278912, "nbytes": 67108864}, {"name": "gate_up_packed#L00004", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 23981387776, "nbytes": 2147483648}, {"name": "gate_up_scale#L00004", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 26128871424, "nbytes": 134217728}, {"name": "down_packed#L00004", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 26263089152, "nbytes": 1073741824}, {"name": "down_scale#L00004", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 27336830976, "nbytes": 67108864}, {"name": "gate_up_packed#L00005", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 27403939840, "nbytes": 2147483648}, {"name": "gate_up_scale#L00005", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 29551423488, "nbytes": 134217728}, {"name": "down_packed#L00005", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 29685641216, "nbytes": 1073741824}, {"name": "down_scale#L00005", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 30759383040, "nbytes": 67108864}, {"name": "gate_up_packed#L00006", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 30826491904, "nbytes": 2147483648}, {"name": "gate_up_scale#L00006", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 32973975552, "nbytes": 134217728}, {"name": "down_packed#L00006", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 33108193280, "nbytes": 1073741824}, {"name": "down_scale#L00006", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 34181935104, "nbytes": 67108864}, {"name": "gate_up_packed#L00007", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 34249043968, "nbytes": 2147483648}, {"name": "gate_up_scale#L00007", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 36396527616, "nbytes": 134217728}, {"name": "down_packed#L00007", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 36530745344, "nbytes": 1073741824}, {"name": "down_scale#L00007", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 37604487168, "nbytes": 67108864}, {"name": "gate_up_packed#L00008", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 37671596032, "nbytes": 2147483648}, {"name": "gate_up_scale#L00008", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 39819079680, "nbytes": 134217728}, {"name": "down_packed#L00008", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 39953297408, "nbytes": 1073741824}, {"name": "down_scale#L00008", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 41027039232, "nbytes": 67108864}, {"name": "gate_up_packed#L00009", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 41094148096, "nbytes": 2147483648}, {"name": "gate_up_scale#L00009", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 43241631744, "nbytes": 134217728}, {"name": "down_packed#L00009", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 43375849472, "nbytes": 1073741824}, {"name": "down_scale#L00009", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 44449591296, "nbytes": 67108864}, {"name": "gate_up_packed#L00010", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 44516700160, "nbytes": 2147483648}, {"name": "gate_up_scale#L00010", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 46664183808, "nbytes": 134217728}, {"name": "down_packed#L00010", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 46798401536, "nbytes": 1073741824}, {"name": "down_scale#L00010", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 47872143360, "nbytes": 67108864}, {"name": "gate_up_packed#L00011", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 47939252224, "nbytes": 2147483648}, {"name": "gate_up_scale#L00011", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 50086735872, "nbytes": 134217728}, {"name": "down_packed#L00011", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 50220953600, "nbytes": 1073741824}, {"name": "down_scale#L00011", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 51294695424, "nbytes": 67108864}, {"name": "gate_up_packed#L00012", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 51361804288, "nbytes": 2147483648}, {"name": "gate_up_scale#L00012", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 53509287936, "nbytes": 134217728}, {"name": "down_packed#L00012", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 53643505664, "nbytes": 1073741824}, {"name": "down_scale#L00012", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 54717247488, "nbytes": 67108864}, {"name": "gate_up_packed#L00013", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 54784356352, "nbytes": 2147483648}, {"name": "gate_up_scale#L00013", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 56931840000, "nbytes": 134217728}, {"name": "down_packed#L00013", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 57066057728, "nbytes": 1073741824}, {"name": "down_scale#L00013", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 58139799552, "nbytes": 67108864}, {"name": "gate_up_packed#L00014", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 58206908416, "nbytes": 2147483648}, {"name": "gate_up_scale#L00014", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 60354392064, "nbytes": 134217728}, {"name": "down_packed#L00014", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 60488609792, "nbytes": 1073741824}, {"name": "down_scale#L00014", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 61562351616, "nbytes": 67108864}, {"name": "gate_up_packed#L00015", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 61629460480, "nbytes": 2147483648}, {"name": "gate_up_scale#L00015", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 63776944128, "nbytes": 134217728}, {"name": "down_packed#L00015", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 63911161856, "nbytes": 1073741824}, {"name": "down_scale#L00015", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 64984903680, "nbytes": 67108864}, {"name": "gate_up_packed#L00016", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 65052012544, "nbytes": 2147483648}, {"name": "gate_up_scale#L00016", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 67199496192, "nbytes": 134217728}, {"name": "down_packed#L00016", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 67333713920, "nbytes": 1073741824}, {"name": "down_scale#L00016", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 68407455744, "nbytes": 67108864}, {"name": "gate_up_packed#L00017", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 68474564608, "nbytes": 2147483648}, {"name": "gate_up_scale#L00017", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 70622048256, "nbytes": 134217728}, {"name": "down_packed#L00017", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 70756265984, "nbytes": 1073741824}, {"name": "down_scale#L00017", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 71830007808, "nbytes": 67108864}, {"name": "gate_up_packed#L00018", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 71897116672, "nbytes": 2147483648}, {"name": "gate_up_scale#L00018", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 74044600320, "nbytes": 134217728}, {"name": "down_packed#L00018", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 74178818048, "nbytes": 1073741824}, {"name": "down_scale#L00018", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 75252559872, "nbytes": 67108864}, {"name": "gate_up_packed#L00019", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 75319668736, "nbytes": 2147483648}, {"name": "gate_up_scale#L00019", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 77467152384, "nbytes": 134217728}, {"name": "down_packed#L00019", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 77601370112, "nbytes": 1073741824}, {"name": "down_scale#L00019", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 78675111936, "nbytes": 67108864}, {"name": "gate_up_packed#L00020", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 78742220800, "nbytes": 2147483648}, {"name": "gate_up_scale#L00020", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 80889704448, "nbytes": 134217728}, {"name": "down_packed#L00020", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 81023922176, "nbytes": 1073741824}, {"name": "down_scale#L00020", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 82097664000, "nbytes": 67108864}, {"name": "gate_up_packed#L00021", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 82164772864, "nbytes": 2147483648}, {"name": "gate_up_scale#L00021", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 84312256512, "nbytes": 134217728}, {"name": "down_packed#L00021", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 84446474240, "nbytes": 1073741824}, {"name": "down_scale#L00021", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 85520216064, "nbytes": 67108864}, {"name": "gate_up_packed#L00022", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 85587324928, "nbytes": 2147483648}, {"name": "gate_up_scale#L00022", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 87734808576, "nbytes": 134217728}, {"name": "down_packed#L00022", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 87869026304, "nbytes": 1073741824}, {"name": "down_scale#L00022", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 88942768128, "nbytes": 67108864}, {"name": "gate_up_packed#L00023", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 89009876992, "nbytes": 2147483648}, {"name": "gate_up_scale#L00023", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 91157360640, "nbytes": 134217728}, {"name": "down_packed#L00023", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 91291578368, "nbytes": 1073741824}, {"name": "down_scale#L00023", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 92365320192, "nbytes": 67108864}, {"name": "gate_up_packed#L00024", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 92432429056, "nbytes": 2147483648}, {"name": "gate_up_scale#L00024", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 94579912704, "nbytes": 134217728}, {"name": "down_packed#L00024", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 94714130432, "nbytes": 1073741824}, {"name": "down_scale#L00024", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 95787872256, "nbytes": 67108864}, {"name": "gate_up_packed#L00025", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 95854981120, "nbytes": 2147483648}, {"name": "gate_up_scale#L00025", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 98002464768, "nbytes": 134217728}, {"name": "down_packed#L00025", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 98136682496, "nbytes": 1073741824}, {"name": "down_scale#L00025", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 99210424320, "nbytes": 67108864}, {"name": "gate_up_packed#L00026", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 99277533184, "nbytes": 2147483648}, {"name": "gate_up_scale#L00026", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 101425016832, "nbytes": 134217728}, {"name": "down_packed#L00026", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 101559234560, "nbytes": 1073741824}, {"name": "down_scale#L00026", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 102632976384, "nbytes": 67108864}, {"name": "gate_up_packed#L00027", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 102700085248, "nbytes": 2147483648}, {"name": "gate_up_scale#L00027", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 104847568896, "nbytes": 134217728}, {"name": "down_packed#L00027", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 104981786624, "nbytes": 1073741824}, {"name": "down_scale#L00027", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 106055528448, "nbytes": 67108864}, {"name": "gate_up_packed#L00028", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 106122637312, "nbytes": 2147483648}, {"name": "gate_up_scale#L00028", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 108270120960, "nbytes": 134217728}, {"name": "down_packed#L00028", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 108404338688, "nbytes": 1073741824}, {"name": "down_scale#L00028", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 109478080512, "nbytes": 67108864}, {"name": "gate_up_packed#L00029", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 109545189376, "nbytes": 2147483648}, {"name": "gate_up_scale#L00029", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 111692673024, "nbytes": 134217728}, {"name": "down_packed#L00029", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 111826890752, "nbytes": 1073741824}, {"name": "down_scale#L00029", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 112900632576, "nbytes": 67108864}, {"name": "gate_up_packed#L00030", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 112967741440, "nbytes": 2147483648}, {"name": "gate_up_scale#L00030", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 115115225088, "nbytes": 134217728}, {"name": "down_packed#L00030", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 115249442816, "nbytes": 1073741824}, {"name": "down_scale#L00030", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 116323184640, "nbytes": 67108864}, {"name": "gate_up_packed#L00031", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 116390293504, "nbytes": 2147483648}, {"name": "gate_up_scale#L00031", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 118537777152, "nbytes": 134217728}, {"name": "down_packed#L00031", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 118671994880, "nbytes": 1073741824}, {"name": "down_scale#L00031", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 119745736704, "nbytes": 67108864}, {"name": "gate_up_packed#L00032", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 119812845568, "nbytes": 2147483648}, {"name": "gate_up_scale#L00032", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 121960329216, "nbytes": 134217728}, {"name": "down_packed#L00032", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 122094546944, "nbytes": 1073741824}, {"name": "down_scale#L00032", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 123168288768, "nbytes": 67108864}, {"name": "gate_up_packed#L00033", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 123235397632, "nbytes": 2147483648}, {"name": "gate_up_scale#L00033", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 125382881280, "nbytes": 134217728}, {"name": "down_packed#L00033", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 125517099008, "nbytes": 1073741824}, {"name": "down_scale#L00033", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 126590840832, "nbytes": 67108864}, {"name": "gate_up_packed#L00034", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 126657949696, "nbytes": 2147483648}, {"name": "gate_up_scale#L00034", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 128805433344, "nbytes": 134217728}, {"name": "down_packed#L00034", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 128939651072, "nbytes": 1073741824}, {"name": "down_scale#L00034", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 130013392896, "nbytes": 67108864}, {"name": "gate_up_packed#L00035", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 130080501760, "nbytes": 2147483648}, {"name": "gate_up_scale#L00035", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 132227985408, "nbytes": 134217728}, {"name": "down_packed#L00035", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 132362203136, "nbytes": 1073741824}, {"name": "down_scale#L00035", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 133435944960, "nbytes": 67108864}, {"name": "gate_up_packed#L00036", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 133503053824, "nbytes": 2147483648}, {"name": "gate_up_scale#L00036", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 135650537472, "nbytes": 134217728}, {"name": "down_packed#L00036", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 135784755200, "nbytes": 1073741824}, {"name": "down_scale#L00036", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 136858497024, "nbytes": 67108864}, {"name": "gate_up_packed#L00037", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 136925605888, "nbytes": 2147483648}, {"name": "gate_up_scale#L00037", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 139073089536, "nbytes": 134217728}, {"name": "down_packed#L00037", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 139207307264, "nbytes": 1073741824}, {"name": "down_scale#L00037", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 140281049088, "nbytes": 67108864}, {"name": "gate_up_packed#L00038", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 140348157952, "nbytes": 2147483648}, {"name": "gate_up_scale#L00038", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 142495641600, "nbytes": 134217728}, {"name": "down_packed#L00038", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 142629859328, "nbytes": 1073741824}, {"name": "down_scale#L00038", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 143703601152, "nbytes": 67108864}, {"name": "gate_up_packed#L00039", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 143770710016, "nbytes": 2147483648}, {"name": "gate_up_scale#L00039", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 145918193664, "nbytes": 134217728}, {"name": "down_packed#L00039", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 146052411392, "nbytes": 1073741824}, {"name": "down_scale#L00039", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 147126153216, "nbytes": 67108864}, {"name": "gate_up_packed#L00040", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 147193262080, "nbytes": 2147483648}, {"name": "gate_up_scale#L00040", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 149340745728, "nbytes": 134217728}, {"name": "down_packed#L00040", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 149474963456, "nbytes": 1073741824}, {"name": "down_scale#L00040", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 150548705280, "nbytes": 67108864}, {"name": "gate_up_packed#L00041", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 150615814144, "nbytes": 2147483648}, {"name": "gate_up_scale#L00041", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 152763297792, "nbytes": 134217728}, {"name": "down_packed#L00041", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 152897515520, "nbytes": 1073741824}, {"name": "down_scale#L00041", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 153971257344, "nbytes": 67108864}, {"name": "gate_up_packed#L00042", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 2048], "global_off": 154038366208, "nbytes": 2147483648}, {"name": "gate_up_scale#L00042", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 128], "global_off": 156185849856, "nbytes": 134217728}, {"name": "down_packed#L00042", "kind": "experts_bank", "dtype": "uint8", "shape": [256, 4096, 1024], "global_off": 156320067584, "nbytes": 1073741824}, {"name": "down_scale#L00042", "kind": "experts_bank", "dtype": "float8_e8m0fnu", "shape": [256, 4096, 64], "global_off": 157393809408, "nbytes": 67108864}], "shards": [{"file": "freetoken-00000.ftw", "global_off": 0, "nbytes": 8572215296}, {"file": "freetoken-00001.ftw", "global_off": 8572215296, "nbytes": 8564068352}, {"file": "freetoken-00002.ftw", "global_off": 17136283648, "nbytes": 6845104128}, {"file": "freetoken-00003.ftw", "global_off": 23981387776, "nbytes": 6845104128}, {"file": "freetoken-00004.ftw", "global_off": 30826491904, "nbytes": 6845104128}, {"file": "freetoken-00005.ftw", "global_off": 37671596032, "nbytes": 6845104128}, {"file": "freetoken-00006.ftw", "global_off": 44516700160, "nbytes": 6845104128}, {"file": "freetoken-00007.ftw", "global_off": 51361804288, "nbytes": 6845104128}, {"file": "freetoken-00008.ftw", "global_off": 58206908416, "nbytes": 6845104128}, {"file": "freetoken-00009.ftw", "global_off": 65052012544, "nbytes": 6845104128}, {"file": "freetoken-00010.ftw", "global_off": 71897116672, "nbytes": 6845104128}, {"file": "freetoken-00011.ftw", "global_off": 78742220800, "nbytes": 6845104128}, {"file": "freetoken-00012.ftw", "global_off": 85587324928, "nbytes": 6845104128}, {"file": "freetoken-00013.ftw", "global_off": 92432429056, "nbytes": 6845104128}, {"file": "freetoken-00014.ftw", "global_off": 99277533184, "nbytes": 6845104128}, {"file": "freetoken-00015.ftw", "global_off": 106122637312, "nbytes": 6845104128}, {"file": "freetoken-00016.ftw", "global_off": 112967741440, "nbytes": 6845104128}, {"file": "freetoken-00017.ftw", "global_off": 119812845568, "nbytes": 6845104128}, {"file": "freetoken-00018.ftw", "global_off": 126657949696, "nbytes": 6845104128}, {"file": "freetoken-00019.ftw", "global_off": 133503053824, "nbytes": 6845104128}, {"file": "freetoken-00020.ftw", "global_off": 140348157952, "nbytes": 6845104128}, {"file": "freetoken-00021.ftw", "global_off": 147193262080, "nbytes": 6845104128}, {"file": "freetoken-00022.ftw", "global_off": 154038366208, "nbytes": 3422552064}], "source_model_path": "/home/lidaiqing/models/DeepSeek-V4-Flash-0731", "fingerprint": "72acedb1d9578b22", "quant_format": "ds_fp4", "expert_bank_num_layers": 43, "counts": {"weight": 1521, "experts_bank": 172}, "copied_metadata": ["README.md", ".gitattributes", "tokenizer_config.json", "generation_config.json", "config.json", "tokenizer.json", "LICENSE", "inference/README.md", "inference/generate.py", "inference/kernel.py", "inference/config.json", "inference/requirements.txt", "inference/convert.py", "inference/model.py", "encoding/README.md", "encoding/test_encoding_dsv4.py", "encoding/encoding_dsv4.py", "encoding/tests/test_input_3.json", "encoding/tests/test_input_4.json", "encoding/tests/test_output_1.txt", "encoding/tests/test_input_2.json", "encoding/tests/test_input_1.json", "encoding/tests/test_output_2.txt", "encoding/tests/test_output_3.txt", "encoding/tests/test_output_4.txt"]}