Hugging Face's logo

Schestex
/
ThinkingCap-Qwen3.8-27B-NInfer

Text Generation
NInfer
qwen.83
qwen3_8
blackwell
sm120a
mtp
speculative-decoding
indexed88
vocabulary-prune
coding
agentic
nvfp4
conversational
Model card Files
xet
 Community
5
ThinkingCap-Qwen3.8-27B-NInfer / bench /mtp1-gpu1-145w-r3.json
Schestex's picture
Schestex
RC1: D48+O44+I53+G64 + native Q8 MTP + Sharp
b78a22f verified
Raw History Blame Contribute Delete
16.1 kB
{
"schema_version": 15,
"artifact_type": "ninfer_bench_report",
"tool": "ninfer_bench",
"command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 1 --lm-head-draft --output json --output-file /out/mtp1-gpu1-r3.json",
"environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0},
"artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292},
"load": {
"architecture": "Qwen3_5ForCausalLM",
"name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8",
"formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"],
"prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc",
"load_seconds": 3.292459731,
"upload_seconds": 2.464175771,
"artifact_bytes_read": 17501379593,
"host_to_device_bytes": 17487933252,
"peak_staging_bytes": 268435456,
"device_object_count": 732,
"host_object_count": 4
},
"memory": {
"device": 0,
"max_context": 16384,
"kv_capacity_mode": "explicit",
"kv_capacity": 16384,
"kv_capacity_page_groups": 256,
"kv_capacity_max_page_groups": 256,
"kv_cache": "fp8-e4m3-row256",
"weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064},
"sequence": {"capacity_bytes": 745966848, "used_bytes": 745966848, "peak_used_bytes": 745966848},
"workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0},
"vision_workspace": null,
"minimum_runtime_reservation_bytes": 991931648,
"kv_capacity_increment_bytes": 0,
"runtime_reservation_bytes": 991931648,
"available_after_weights_bytes": 7420313600,
"available_after_startup_bytes": 6510149632,
"kv_capacity_headroom_bytes": 0,
"planned_slack_bytes": 6428381952,
"cuda_graph_allowance_bytes": 85983232,
"kv_payload_bytes": 574881792
},
"config": {
"max_context": 16384,
"prefill_chunk": 1024,
"kv_cache": "fp8-e4m3-row256",
"speculative_backend": "mtp",
"draft_tokens": 1,
"proposal_head": "optimized",
"use_cuda_graph": true,
"decode_path": "mtp_cuda_graph",
"decode_graph_prime": {"primed": true, "output_tokens": 5},
"repetitions": 3,
"warmup": 2,
"corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids",
"corpus_tokens": 65536
},
"tests": [
{
"label": "tg256",
"kind": "tg",
"n_prompt": 0,
"n_gen": 256,
"requested_output_tokens": 257,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 48.45010013,
"decode_output_tok_s_stddev": 0.02528587508,
"decode_engine_tok_s_mean": 48.45010013,
"decode_engine_tok_s_stddev": 0.02528587508,
"prepare_seconds_mean": 1.91e-06,
"prepare_seconds_stddev": 5.022947342e-07,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 5.283787769,
"decode_seconds_stddev": 0.00275703966,
"total_seconds_mean": 5.315449896,
"total_seconds_stddev": 0.002734959798,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 480,
"drafted_tokens": 480,
"accepted_tokens": 285,
"fallback_steps": 3,
"acceptance_rate": 0.59375,
"acceptance_length": 1.59375,
"accepted_per_position": [285]
},
"reps": [
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 2.49e-06, "vision_seconds": 0, "prefill_seconds": 0.031382741, "decode_seconds": 5.28073335, "total_seconds": 5.312406691},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 160,
"drafted_tokens": 160,
"accepted_tokens": 95,
"fallback_steps": 1,
"acceptance_rate": 0.59375,
"acceptance_length": 1.59375,
"accepted_per_position": [95]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.62e-06, "vision_seconds": 0, "prefill_seconds": 0.031418811, "decode_seconds": 5.284537699, "total_seconds": 5.316240619},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 160,
"drafted_tokens": 160,
"accepted_tokens": 95,
"fallback_steps": 1,
"acceptance_rate": 0.59375,
"acceptance_length": 1.59375,
"accepted_per_position": [95]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.62e-06, "vision_seconds": 0, "prefill_seconds": 0.03132409, "decode_seconds": 5.286092257, "total_seconds": 5.317702377},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 160,
"drafted_tokens": 160,
"accepted_tokens": 95,
"fallback_steps": 1,
"acceptance_rate": 0.59375,
"acceptance_length": 1.59375,
"accepted_per_position": [95]
}
}
]
},
{
"label": "tg1024",
"kind": "tg",
"n_prompt": 0,
"n_gen": 1024,
"requested_output_tokens": 1025,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 52.37556268,
"decode_output_tok_s_stddev": 0.04727580299,
"decode_engine_tok_s_mean": 52.37556268,
"decode_engine_tok_s_stddev": 0.04727580299,
"prepare_seconds_mean": 2.22e-06,
"prepare_seconds_stddev": 4.33474336e-07,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 19.55111323,
"decode_seconds_stddev": 0.01765105117,
"total_seconds_mean": 19.58346822,
"total_seconds_stddev": 0.01764927456,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 1782,
"drafted_tokens": 1782,
"accepted_tokens": 1290,
"fallback_steps": 0,
"acceptance_rate": 0.7239057239,
"acceptance_length": 1.723905724,
"accepted_per_position": [1290]
},
"reps": [
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 1.95e-06, "vision_seconds": 0, "prefill_seconds": 0.03139705, "decode_seconds": 19.53499106, "total_seconds": 19.56735712},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 594,
"drafted_tokens": 594,
"accepted_tokens": 430,
"fallback_steps": 0,
"acceptance_rate": 0.7239057239,
"acceptance_length": 1.723905724,
"accepted_per_position": [430]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 2.72e-06, "vision_seconds": 0, "prefill_seconds": 0.03139285, "decode_seconds": 19.5483754, "total_seconds": 19.58071538},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 594,
"drafted_tokens": 594,
"accepted_tokens": 430,
"fallback_steps": 0,
"acceptance_rate": 0.7239057239,
"acceptance_length": 1.723905724,
"accepted_per_position": [430]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 1.99e-06, "vision_seconds": 0, "prefill_seconds": 0.03141448, "decode_seconds": 19.56997322, "total_seconds": 19.60233216},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 594,
"drafted_tokens": 594,
"accepted_tokens": 430,
"fallback_steps": 0,
"acceptance_rate": 0.7239057239,
"acceptance_length": 1.723905724,
"accepted_per_position": [430]
}
}
]
},
{
"label": "pp2048+tg512",
"kind": "pp+tg",
"n_prompt": 2048,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3616.226848,
"prefill_tok_s_stddev": 1.239296621,
"decode_output_tok_s_mean": 59.05174916,
"decode_output_tok_s_stddev": 0.008858635486,
"decode_engine_tok_s_mean": 59.05174916,
"decode_engine_tok_s_stddev": 0.008858635486,
"prepare_seconds_mean": 1.011666667e-05,
"prepare_seconds_stddev": 2.881411691e-06,
"prefill_seconds_mean": 0.566336197,
"prefill_seconds_stddev": 0.0001940910284,
"decode_seconds_mean": 8.67036142,
"decode_seconds_stddev": 0.001300582635,
"total_seconds_mean": 9.237204947,
"total_seconds_stddev": 0.001274274554,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 780,
"drafted_tokens": 780,
"accepted_tokens": 753,
"fallback_steps": 3,
"acceptance_rate": 0.9653846154,
"acceptance_length": 1.965384615,
"accepted_per_position": [753]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 6.87e-06, "vision_seconds": 0, "prefill_seconds": 0.566535133, "decode_seconds": 8.67089422, "total_seconds": 9.237908823},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 260,
"drafted_tokens": 260,
"accepted_tokens": 251,
"fallback_steps": 1,
"acceptance_rate": 0.9653846154,
"acceptance_length": 1.965384615,
"accepted_per_position": [251]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 1.111e-05, "vision_seconds": 0, "prefill_seconds": 0.566326114, "decode_seconds": 8.66887904, "total_seconds": 9.235733994},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 260,
"drafted_tokens": 260,
"accepted_tokens": 251,
"fallback_steps": 1,
"acceptance_rate": 0.9653846154,
"acceptance_length": 1.965384615,
"accepted_per_position": [251]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 1.237e-05, "vision_seconds": 0, "prefill_seconds": 0.566147344, "decode_seconds": 8.671311001, "total_seconds": 9.237972024},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 260,
"drafted_tokens": 260,
"accepted_tokens": 251,
"fallback_steps": 1,
"acceptance_rate": 0.9653846154,
"acceptance_length": 1.965384615,
"accepted_per_position": [251]
}
}
]
},
{
"label": "pp8192+tg512",
"kind": "pp+tg",
"n_prompt": 8192,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3441.180646,
"prefill_tok_s_stddev": 1.407556413,
"decode_output_tok_s_mean": 59.31600179,
"decode_output_tok_s_stddev": 0.009298871911,
"decode_engine_tok_s_mean": 59.31600179,
"decode_engine_tok_s_stddev": 0.009298871911,
"prepare_seconds_mean": 2.128333333e-05,
"prepare_seconds_stddev": 5.836654293e-06,
"prefill_seconds_mean": 2.380578573,
"prefill_seconds_stddev": 0.0009735672692,
"decode_seconds_mean": 8.631734994,
"decode_seconds_stddev": 0.00135325813,
"total_seconds_mean": 11.01281932,
"total_seconds_stddev": 0.001979281361,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 771,
"drafted_tokens": 771,
"accepted_tokens": 765,
"fallback_steps": 0,
"acceptance_rate": 0.9922178988,
"acceptance_length": 1.992217899,
"accepted_per_position": [765]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 1.721e-05, "vision_seconds": 0, "prefill_seconds": 2.381364663, "decode_seconds": 8.633226403, "total_seconds": 11.01507785},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 257,
"drafted_tokens": 257,
"accepted_tokens": 255,
"fallback_steps": 0,
"acceptance_rate": 0.9922178988,
"acceptance_length": 1.992217899,
"accepted_per_position": [255]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.797e-05, "vision_seconds": 0, "prefill_seconds": 2.380881503, "decode_seconds": 8.630585455, "total_seconds": 11.01199311},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 257,
"drafted_tokens": 257,
"accepted_tokens": 255,
"fallback_steps": 0,
"acceptance_rate": 0.9922178988,
"acceptance_length": 1.992217899,
"accepted_per_position": [255]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 1.867e-05, "vision_seconds": 0, "prefill_seconds": 2.379489554, "decode_seconds": 8.631393125, "total_seconds": 11.011387},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 1,
"rounds": 257,
"drafted_tokens": 257,
"accepted_tokens": 255,
"fallback_steps": 0,
"acceptance_rate": 0.9922178988,
"acceptance_length": 1.992217899,
"accepted_per_position": [255]
}
}
]
}
]
}