| { |
| "metadata": { |
| "ParamSize": 76, |
| "ParamBytes": 281706496.0, |
| "BitsPerParam": 32.0 |
| }, |
| "records": [ |
| { |
| "dataPath": "params_shard_0.bin", |
| "format": "raw-shard", |
| "nbytes": 103022592, |
| "records": [ |
| { |
| "name": "embed_out.weight", |
| "shape": [ |
| 50304, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 103022592, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "0a623ea8144d86c80bdf32b3f20f378c" |
| }, |
| { |
| "dataPath": "params_shard_1.bin", |
| "format": "raw-shard", |
| "nbytes": 103022592, |
| "records": [ |
| { |
| "name": "gpt_neox.embed_in.weight", |
| "shape": [ |
| 50304, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 103022592, |
| "byteOffset": 0 |
| } |
| ], |
| "md5sum": "f945d3830b66ed726b53093acf4b7510" |
| }, |
| { |
| "dataPath": "params_shard_2.bin", |
| "format": "raw-shard", |
| "nbytes": 29431808, |
| "records": [ |
| { |
| "name": "gpt_neox.final_layer_norm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "gpt_neox.final_layer_norm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 2048 |
| }, |
| { |
| "name": "gpt_neox.layers.0.attention.dense.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 4096 |
| }, |
| { |
| "name": "gpt_neox.layers.0.attention.dense.weight", |
| "shape": [ |
| 512, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1048576, |
| "byteOffset": 6144 |
| }, |
| { |
| "name": "gpt_neox.layers.0.attention.query_key_value.bias", |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 6144, |
| "byteOffset": 1054720 |
| }, |
| { |
| "name": "gpt_neox.layers.0.attention.query_key_value.weight", |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 3145728, |
| "byteOffset": 1060864 |
| }, |
| { |
| "name": "gpt_neox.layers.0.input_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 4206592 |
| }, |
| { |
| "name": "gpt_neox.layers.0.input_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 4208640 |
| }, |
| { |
| "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 4210688 |
| }, |
| { |
| "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight", |
| "shape": [ |
| 512, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 4212736 |
| }, |
| { |
| "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 8192, |
| "byteOffset": 8407040 |
| }, |
| { |
| "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight", |
| "shape": [ |
| 2048, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 8415232 |
| }, |
| { |
| "name": "gpt_neox.layers.0.post_attention_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 12609536 |
| }, |
| { |
| "name": "gpt_neox.layers.0.post_attention_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 12611584 |
| }, |
| { |
| "name": "gpt_neox.layers.1.attention.dense.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 12613632 |
| }, |
| { |
| "name": "gpt_neox.layers.1.attention.dense.weight", |
| "shape": [ |
| 512, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1048576, |
| "byteOffset": 12615680 |
| }, |
| { |
| "name": "gpt_neox.layers.1.attention.query_key_value.bias", |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 6144, |
| "byteOffset": 13664256 |
| }, |
| { |
| "name": "gpt_neox.layers.1.attention.query_key_value.weight", |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 3145728, |
| "byteOffset": 13670400 |
| }, |
| { |
| "name": "gpt_neox.layers.1.input_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 16816128 |
| }, |
| { |
| "name": "gpt_neox.layers.1.input_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 16818176 |
| }, |
| { |
| "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 16820224 |
| }, |
| { |
| "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight", |
| "shape": [ |
| 512, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 16822272 |
| }, |
| { |
| "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 8192, |
| "byteOffset": 21016576 |
| }, |
| { |
| "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight", |
| "shape": [ |
| 2048, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 21024768 |
| }, |
| { |
| "name": "gpt_neox.layers.1.post_attention_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 25219072 |
| }, |
| { |
| "name": "gpt_neox.layers.1.post_attention_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 25221120 |
| }, |
| { |
| "name": "gpt_neox.layers.2.attention.dense.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 25223168 |
| }, |
| { |
| "name": "gpt_neox.layers.2.attention.dense.weight", |
| "shape": [ |
| 512, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1048576, |
| "byteOffset": 25225216 |
| }, |
| { |
| "name": "gpt_neox.layers.2.attention.query_key_value.bias", |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 6144, |
| "byteOffset": 26273792 |
| }, |
| { |
| "name": "gpt_neox.layers.2.attention.query_key_value.weight", |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 3145728, |
| "byteOffset": 26279936 |
| }, |
| { |
| "name": "gpt_neox.layers.2.input_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 29425664 |
| }, |
| { |
| "name": "gpt_neox.layers.2.input_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 29427712 |
| }, |
| { |
| "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 29429760 |
| } |
| ], |
| "md5sum": "a5b014faa17e4f1a8e65a813eb0dfb3a" |
| }, |
| { |
| "dataPath": "params_shard_3.bin", |
| "format": "raw-shard", |
| "nbytes": 29421568, |
| "records": [ |
| { |
| "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight", |
| "shape": [ |
| 512, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 8192, |
| "byteOffset": 4194304 |
| }, |
| { |
| "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight", |
| "shape": [ |
| 2048, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 4202496 |
| }, |
| { |
| "name": "gpt_neox.layers.2.post_attention_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 8396800 |
| }, |
| { |
| "name": "gpt_neox.layers.2.post_attention_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 8398848 |
| }, |
| { |
| "name": "gpt_neox.layers.3.attention.dense.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 8400896 |
| }, |
| { |
| "name": "gpt_neox.layers.3.attention.dense.weight", |
| "shape": [ |
| 512, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1048576, |
| "byteOffset": 8402944 |
| }, |
| { |
| "name": "gpt_neox.layers.3.attention.query_key_value.bias", |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 6144, |
| "byteOffset": 9451520 |
| }, |
| { |
| "name": "gpt_neox.layers.3.attention.query_key_value.weight", |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 3145728, |
| "byteOffset": 9457664 |
| }, |
| { |
| "name": "gpt_neox.layers.3.input_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 12603392 |
| }, |
| { |
| "name": "gpt_neox.layers.3.input_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 12605440 |
| }, |
| { |
| "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 12607488 |
| }, |
| { |
| "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight", |
| "shape": [ |
| 512, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 12609536 |
| }, |
| { |
| "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 8192, |
| "byteOffset": 16803840 |
| }, |
| { |
| "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight", |
| "shape": [ |
| 2048, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 16812032 |
| }, |
| { |
| "name": "gpt_neox.layers.3.post_attention_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 21006336 |
| }, |
| { |
| "name": "gpt_neox.layers.3.post_attention_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 21008384 |
| }, |
| { |
| "name": "gpt_neox.layers.4.attention.dense.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 21010432 |
| }, |
| { |
| "name": "gpt_neox.layers.4.attention.dense.weight", |
| "shape": [ |
| 512, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1048576, |
| "byteOffset": 21012480 |
| }, |
| { |
| "name": "gpt_neox.layers.4.attention.query_key_value.bias", |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 6144, |
| "byteOffset": 22061056 |
| }, |
| { |
| "name": "gpt_neox.layers.4.attention.query_key_value.weight", |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 3145728, |
| "byteOffset": 22067200 |
| }, |
| { |
| "name": "gpt_neox.layers.4.input_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 25212928 |
| }, |
| { |
| "name": "gpt_neox.layers.4.input_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 25214976 |
| }, |
| { |
| "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 25217024 |
| }, |
| { |
| "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight", |
| "shape": [ |
| 512, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 25219072 |
| }, |
| { |
| "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 8192, |
| "byteOffset": 29413376 |
| } |
| ], |
| "md5sum": "8938b1b2f84660600e662265cf728153" |
| }, |
| { |
| "dataPath": "params_shard_4.bin", |
| "format": "raw-shard", |
| "nbytes": 16807936, |
| "records": [ |
| { |
| "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight", |
| "shape": [ |
| 2048, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 0 |
| }, |
| { |
| "name": "gpt_neox.layers.4.post_attention_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 4194304 |
| }, |
| { |
| "name": "gpt_neox.layers.4.post_attention_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 4196352 |
| }, |
| { |
| "name": "gpt_neox.layers.5.attention.dense.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 4198400 |
| }, |
| { |
| "name": "gpt_neox.layers.5.attention.dense.weight", |
| "shape": [ |
| 512, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 1048576, |
| "byteOffset": 4200448 |
| }, |
| { |
| "name": "gpt_neox.layers.5.attention.query_key_value.bias", |
| "shape": [ |
| 1536 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 6144, |
| "byteOffset": 5249024 |
| }, |
| { |
| "name": "gpt_neox.layers.5.attention.query_key_value.weight", |
| "shape": [ |
| 1536, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 3145728, |
| "byteOffset": 5255168 |
| }, |
| { |
| "name": "gpt_neox.layers.5.input_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 8400896 |
| }, |
| { |
| "name": "gpt_neox.layers.5.input_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 8402944 |
| }, |
| { |
| "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 8404992 |
| }, |
| { |
| "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight", |
| "shape": [ |
| 512, |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 8407040 |
| }, |
| { |
| "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", |
| "shape": [ |
| 2048 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 8192, |
| "byteOffset": 12601344 |
| }, |
| { |
| "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight", |
| "shape": [ |
| 2048, |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 4194304, |
| "byteOffset": 12609536 |
| }, |
| { |
| "name": "gpt_neox.layers.5.post_attention_layernorm.bias", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 16803840 |
| }, |
| { |
| "name": "gpt_neox.layers.5.post_attention_layernorm.weight", |
| "shape": [ |
| 512 |
| ], |
| "dtype": "float32", |
| "format": "raw", |
| "nbytes": 2048, |
| "byteOffset": 16805888 |
| } |
| ], |
| "md5sum": "1647274ac091bba533b982ed2c89a91b" |
| } |
| ] |
| } |
