| { |
| "source": "Contrastive-LM/CLM-v0.1-8B encoder = Qwen/Qwen3-8B", |
| "source_revision": "b968826d9c46dd6066d109eabc6255188de91218", |
| "tool": "mlx-lm 0.31.3 (mlx_lm.convert)", |
| "command": "mlx_lm.convert --hf-path Qwen/Qwen3-8B --mlx-path <dir> -q --q-bits N --q-group-size 32", |
| "mode": "affine, per-group bf16 scale and bias", |
| "group_size": 32, |
| "group_size_note": "32 is the published recipe and measured best at every width; mlx-lm's default (64) and Qwen's mlx-community repos (128) are both worse here. MLX affine spends 32 bits per group on a bf16 scale AND bias, so group size is a first-class quality knob.", |
| "quantised": "every linear weight, including embed_tokens and lm_head; all of Qwen3-8B's quantized last dims (4096, 1024, 12288) divide 32, so nothing is skipped", |
| "lm_head": "present in the safetensors for layout compatibility but replaced with an Identity before evaluation by clm_mlx.Encoder -- pooling never touches it", |
| "sizes_bytes": { |
| "4bit": 5131300029, |
| "6bit": 7178920204, |
| "8bit": 9226527200, |
| "bf16": "unquantised reference, ~15 GB" |
| }, |
| "sizes_note": "8190427136 x (bits + 32/group_size) / 8 + 616448 at group_size 32, i.e. bits + 1.0. Measured, not predicted: mlx_lm.convert reports 9.0117 / 7.0117 / 5.0118 bits per weight.", |
| "variants": { |
| "bf16": { |
| "level": "reference", |
| "recommended": true, |
| "reason": "this is the bf16 baseline every other row is measured against; it is not itself a quantisation result", |
| "top1": null, |
| "decisive_top1": null, |
| "gold_delta": null |
| }, |
| "8bit": { |
| "level": "recommended", |
| "recommended": true, |
| "reason": "meets the gate", |
| "top1": 0.988757, |
| "decisive_top1": 1.0, |
| "decisive_n": 7440, |
| "gold_delta": -0.004569, |
| "gold_loss": 0.004569, |
| "gold_tolerance": 0.005, |
| "floor": 1.0 |
| }, |
| "6bit": { |
| "level": "recommended", |
| "recommended": true, |
| "reason": "meets the gate", |
| "top1": 0.98278, |
| "decisive_top1": 1.0, |
| "decisive_n": 7440, |
| "gold_delta": 0.006958, |
| "gold_loss": -0.006958, |
| "gold_tolerance": 0.005, |
| "floor": 1.0 |
| }, |
| "4bit": { |
| "level": "not_recommended", |
| "recommended": false, |
| "reason": "agreement on decisive (>1 nat) decisions 0.889651 < 0.995; planner accuracy falls 31.95 points, beyond the 3.0-point budget", |
| "top1": 0.618616, |
| "decisive_top1": 0.889651, |
| "decisive_n": 7440, |
| "gold_delta": -0.319515, |
| "gold_loss": 0.319515, |
| "floor": 1.0 |
| } |
| } |
| } |
|
|
