| { | |
| "model_id": "Qwen/Qwen3-0.6B", | |
| "model_family": "qwen3_0_6b", | |
| "num_layers": 28, | |
| "hidden_size": 1024, | |
| "intermediate": 3072, | |
| "vocab_size": 151936, | |
| "num_feats": 128, | |
| "dtype": "float16", | |
| "layers": [ | |
| { | |
| "layer": 0, | |
| "num_features": 128, | |
| "gate_var": 0.3578, | |
| "down_var": 0.2797 | |
| }, | |
| { | |
| "layer": 1, | |
| "num_features": 128, | |
| "gate_var": 0.3216, | |
| "down_var": 0.2958 | |
| }, | |
| { | |
| "layer": 2, | |
| "num_features": 128, | |
| "gate_var": 0.3405, | |
| "down_var": 0.2877 | |
| }, | |
| { | |
| "layer": 3, | |
| "num_features": 128, | |
| "gate_var": 0.3053, | |
| "down_var": 0.2948 | |
| }, | |
| { | |
| "layer": 4, | |
| "num_features": 128, | |
| "gate_var": 0.3214, | |
| "down_var": 0.2927 | |
| }, | |
| { | |
| "layer": 5, | |
| "num_features": 128, | |
| "gate_var": 0.3163, | |
| "down_var": 0.2938 | |
| }, | |
| { | |
| "layer": 6, | |
| "num_features": 128, | |
| "gate_var": 0.3251, | |
| "down_var": 0.3244 | |
| }, | |
| { | |
| "layer": 7, | |
| "num_features": 128, | |
| "gate_var": 0.337, | |
| "down_var": 0.3188 | |
| }, | |
| { | |
| "layer": 8, | |
| "num_features": 128, | |
| "gate_var": 0.336, | |
| "down_var": 0.3376 | |
| }, | |
| { | |
| "layer": 9, | |
| "num_features": 128, | |
| "gate_var": 0.3591, | |
| "down_var": 0.3356 | |
| }, | |
| { | |
| "layer": 10, | |
| "num_features": 128, | |
| "gate_var": 0.357, | |
| "down_var": 0.3267 | |
| }, | |
| { | |
| "layer": 11, | |
| "num_features": 128, | |
| "gate_var": 0.3529, | |
| "down_var": 0.3523 | |
| }, | |
| { | |
| "layer": 12, | |
| "num_features": 128, | |
| "gate_var": 0.3612, | |
| "down_var": 0.3716 | |
| }, | |
| { | |
| "layer": 13, | |
| "num_features": 128, | |
| "gate_var": 0.3779, | |
| "down_var": 0.3799 | |
| }, | |
| { | |
| "layer": 14, | |
| "num_features": 128, | |
| "gate_var": 0.3808, | |
| "down_var": 0.3841 | |
| }, | |
| { | |
| "layer": 15, | |
| "num_features": 128, | |
| "gate_var": 0.3795, | |
| "down_var": 0.3788 | |
| }, | |
| { | |
| "layer": 16, | |
| "num_features": 128, | |
| "gate_var": 0.3745, | |
| "down_var": 0.3475 | |
| }, | |
| { | |
| "layer": 17, | |
| "num_features": 128, | |
| "gate_var": 0.355, | |
| "down_var": 0.3391 | |
| }, | |
| { | |
| "layer": 18, | |
| "num_features": 128, | |
| "gate_var": 0.3512, | |
| "down_var": 0.3443 | |
| }, | |
| { | |
| "layer": 19, | |
| "num_features": 128, | |
| "gate_var": 0.3537, | |
| "down_var": 0.3407 | |
| }, | |
| { | |
| "layer": 20, | |
| "num_features": 128, | |
| "gate_var": 0.3421, | |
| "down_var": 0.3201 | |
| }, | |
| { | |
| "layer": 21, | |
| "num_features": 128, | |
| "gate_var": 0.3387, | |
| "down_var": 0.3039 | |
| }, | |
| { | |
| "layer": 22, | |
| "num_features": 128, | |
| "gate_var": 0.3284, | |
| "down_var": 0.3035 | |
| }, | |
| { | |
| "layer": 23, | |
| "num_features": 128, | |
| "gate_var": 0.3236, | |
| "down_var": 0.3111 | |
| }, | |
| { | |
| "layer": 24, | |
| "num_features": 128, | |
| "gate_var": 0.32, | |
| "down_var": 0.3115 | |
| }, | |
| { | |
| "layer": 25, | |
| "num_features": 128, | |
| "gate_var": 0.3265, | |
| "down_var": 0.3009 | |
| }, | |
| { | |
| "layer": 26, | |
| "num_features": 128, | |
| "gate_var": 0.3449, | |
| "down_var": 0.3025 | |
| }, | |
| { | |
| "layer": 27, | |
| "num_features": 128, | |
| "gate_var": 0.3987, | |
| "down_var": 0.2917 | |
| } | |
| ] | |
| } |