File size: 2,565 Bytes
c040220 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | {
"source": "Contrastive-LM/CLM-v0.1-8B encoder = Qwen/Qwen3-8B",
"source_revision": "b968826d9c46dd6066d109eabc6255188de91218",
"tool": "mlx-lm 0.31.3 (mlx_lm.convert)",
"command": "mlx_lm.convert --hf-path Qwen/Qwen3-8B --mlx-path <dir> -q --q-bits N --q-group-size 32",
"mode": "affine, per-group bf16 scale and bias",
"group_size": 32,
"group_size_note": "32 is the published recipe and measured best at every width; mlx-lm's default (64) and Qwen's mlx-community repos (128) are both worse here. MLX affine spends 32 bits per group on a bf16 scale AND bias, so group size is a first-class quality knob.",
"quantised": "every linear weight, including embed_tokens and lm_head; all of Qwen3-8B's quantized last dims (4096, 1024, 12288) divide 32, so nothing is skipped",
"lm_head": "present in the safetensors for layout compatibility but replaced with an Identity before evaluation by clm_mlx.Encoder -- pooling never touches it",
"sizes_bytes": {
"4bit": 5131300029,
"6bit": 7178920204,
"8bit": 9226527200,
"bf16": "unquantised reference, ~15 GB"
},
"sizes_note": "8190427136 x (bits + 32/group_size) / 8 + 616448 at group_size 32, i.e. bits + 1.0. Measured, not predicted: mlx_lm.convert reports 9.0117 / 7.0117 / 5.0118 bits per weight.",
"variants": {
"bf16": {
"level": "reference",
"recommended": true,
"reason": "this is the bf16 baseline every other row is measured against; it is not itself a quantisation result",
"top1": null,
"decisive_top1": null,
"gold_delta": null
},
"8bit": {
"level": "recommended",
"recommended": true,
"reason": "meets the gate",
"top1": 0.988757,
"decisive_top1": 1.0,
"decisive_n": 7440,
"gold_delta": -0.004569,
"gold_loss": 0.004569,
"gold_tolerance": 0.005,
"floor": 1.0
},
"6bit": {
"level": "recommended",
"recommended": true,
"reason": "meets the gate",
"top1": 0.98278,
"decisive_top1": 1.0,
"decisive_n": 7440,
"gold_delta": 0.006958,
"gold_loss": -0.006958,
"gold_tolerance": 0.005,
"floor": 1.0
},
"4bit": {
"level": "not_recommended",
"recommended": false,
"reason": "agreement on decisive (>1 nat) decisions 0.889651 < 0.995; planner accuracy falls 31.95 points, beyond the 3.0-point budget",
"top1": 0.618616,
"decisive_top1": 0.889651,
"decisive_n": 7440,
"gold_delta": -0.319515,
"gold_loss": 0.319515,
"floor": 1.0
}
}
}
|