Hugging Face's logo

Schestex
/
ThinkingCap-Qwen3.8-27B-NInfer

Text Generation
NInfer
qwen.83
qwen3_8
blackwell
sm120a
mtp
speculative-decoding
indexed88
vocabulary-prune
coding
agentic
nvfp4
conversational
Model card Files
xet
 Community
5
ThinkingCap-Qwen3.8-27B-NInfer / bench /mtp5-gpu1-145w-r3.json
Schestex's picture
Schestex
RC1: D48+O44+I53+G64 + native Q8 MTP + Sharp
b78a22f verified
Raw History Blame Contribute Delete
16.4 kB
{
"schema_version": 15,
"artifact_type": "ninfer_bench_report",
"tool": "ninfer_bench",
"command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 5 --lm-head-draft --output json --output-file /out/mtp5-gpu1-r3.json",
"environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0},
"artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292},
"load": {
"architecture": "Qwen3_5ForCausalLM",
"name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8",
"formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"],
"prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc",
"load_seconds": 3.361672471,
"upload_seconds": 2.471311519,
"artifact_bytes_read": 17501379593,
"host_to_device_bytes": 17487933252,
"peak_staging_bytes": 268435456,
"device_object_count": 732,
"host_object_count": 4
},
"memory": {
"device": 0,
"max_context": 16384,
"kv_capacity_mode": "explicit",
"kv_capacity": 16384,
"kv_capacity_page_groups": 256,
"kv_capacity_max_page_groups": 256,
"kv_cache": "fp8-e4m3-row256",
"weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064},
"sequence": {"capacity_bytes": 755319040, "used_bytes": 755319040, "peak_used_bytes": 755319040},
"workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0},
"vision_workspace": null,
"minimum_runtime_reservation_bytes": 1001283840,
"kv_capacity_increment_bytes": 0,
"runtime_reservation_bytes": 1001283840,
"available_after_weights_bytes": 7420313600,
"available_after_startup_bytes": 6497566720,
"kv_capacity_headroom_bytes": 0,
"planned_slack_bytes": 6419029760,
"cuda_graph_allowance_bytes": 85983232,
"kv_payload_bytes": 575013888
},
"config": {
"max_context": 16384,
"prefill_chunk": 1024,
"kv_cache": "fp8-e4m3-row256",
"speculative_backend": "mtp",
"draft_tokens": 5,
"proposal_head": "optimized",
"use_cuda_graph": true,
"decode_path": "mtp_cuda_graph",
"decode_graph_prime": {"primed": true, "output_tokens": 13},
"repetitions": 3,
"warmup": 2,
"corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids",
"corpus_tokens": 65536
},
"tests": [
{
"label": "tg256",
"kind": "tg",
"n_prompt": 0,
"n_gen": 256,
"requested_output_tokens": 257,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 45.64353647,
"decode_output_tok_s_stddev": 0.01225268331,
"decode_engine_tok_s_mean": 45.64353647,
"decode_engine_tok_s_stddev": 0.01225268331,
"prepare_seconds_mean": 1.626666667e-06,
"prepare_seconds_stddev": 4.163331999e-08,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 5.608680486,
"decode_seconds_stddev": 0.001505812499,
"total_seconds_mean": 5.646164623,
"total_seconds_stddev": 0.001530725286,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 366,
"drafted_tokens": 1818,
"accepted_tokens": 399,
"fallback_steps": 3,
"acceptance_rate": 0.2194719472,
"acceptance_length": 2.090163934,
"accepted_per_position": [195, 111, 60, 27, 6]
},
"reps": [
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.64e-06, "vision_seconds": 0, "prefill_seconds": 0.037218125, "decode_seconds": 5.608086056, "total_seconds": 5.645572101},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 122,
"drafted_tokens": 606,
"accepted_tokens": 133,
"fallback_steps": 1,
"acceptance_rate": 0.2194719472,
"acceptance_length": 2.090163934,
"accepted_per_position": [65, 37, 20, 9, 2]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.58e-06, "vision_seconds": 0, "prefill_seconds": 0.037194564, "decode_seconds": 5.607562617, "total_seconds": 5.64501873},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 122,
"drafted_tokens": 606,
"accepted_tokens": 133,
"fallback_steps": 1,
"acceptance_rate": 0.2194719472,
"acceptance_length": 2.090163934,
"accepted_per_position": [65, 37, 20, 9, 2]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.66e-06, "vision_seconds": 0, "prefill_seconds": 0.037246264, "decode_seconds": 5.610392784, "total_seconds": 5.647903038},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 122,
"drafted_tokens": 606,
"accepted_tokens": 133,
"fallback_steps": 1,
"acceptance_rate": 0.2194719472,
"acceptance_length": 2.090163934,
"accepted_per_position": [65, 37, 20, 9, 2]
}
}
]
},
{
"label": "tg1024",
"kind": "tg",
"n_prompt": 0,
"n_gen": 1024,
"requested_output_tokens": 1025,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 79.96223629,
"decode_output_tok_s_stddev": 0.01928508403,
"decode_engine_tok_s_mean": 79.96223629,
"decode_engine_tok_s_stddev": 0.01928508403,
"prepare_seconds_mean": 1.86e-06,
"prepare_seconds_stddev": 2.443358345e-07,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 12.80604554,
"decode_seconds_stddev": 0.003088215596,
"total_seconds_mean": 12.84390952,
"total_seconds_stddev": 0.003108343472,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 837,
"drafted_tokens": 4185,
"accepted_tokens": 2232,
"fallback_steps": 3,
"acceptance_rate": 0.5333333333,
"acceptance_length": 3.666666667,
"accepted_per_position": [636, 495, 405, 360, 336]
},
"reps": [
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 1.58e-06, "vision_seconds": 0, "prefill_seconds": 0.037275024, "decode_seconds": 12.80854159, "total_seconds": 12.84644666},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 279,
"drafted_tokens": 1395,
"accepted_tokens": 744,
"fallback_steps": 1,
"acceptance_rate": 0.5333333333,
"acceptance_length": 3.666666667,
"accepted_per_position": [212, 165, 135, 120, 112]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 1.97e-06, "vision_seconds": 0, "prefill_seconds": 0.037306394, "decode_seconds": 12.80700306, "total_seconds": 12.84483958},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 279,
"drafted_tokens": 1395,
"accepted_tokens": 744,
"fallback_steps": 1,
"acceptance_rate": 0.5333333333,
"acceptance_length": 3.666666667,
"accepted_per_position": [212, 165, 135, 120, 112]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 2.03e-06, "vision_seconds": 0, "prefill_seconds": 0.037319024, "decode_seconds": 12.80259198, "total_seconds": 12.84044232},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 279,
"drafted_tokens": 1395,
"accepted_tokens": 744,
"fallback_steps": 1,
"acceptance_rate": 0.5333333333,
"acceptance_length": 3.666666667,
"accepted_per_position": [212, 165, 135, 120, 112]
}
}
]
},
{
"label": "pp2048+tg512",
"kind": "pp+tg",
"n_prompt": 2048,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3578.647259,
"prefill_tok_s_stddev": 2.500665781,
"decode_output_tok_s_mean": 116.450161,
"decode_output_tok_s_stddev": 0.01153414308,
"decode_engine_tok_s_mean": 116.450161,
"decode_engine_tok_s_stddev": 0.01153414308,
"prepare_seconds_mean": 8.87e-06,
"prepare_seconds_stddev": 3.304542328e-07,
"prefill_seconds_mean": 0.5722834687,
"prefill_seconds_stddev": 0.0003998407674,
"decode_seconds_mean": 4.396730747,
"decode_seconds_stddev": 0.0004354861691,
"total_seconds_mean": 4.969295035,
"total_seconds_stddev": 0.0008332372725,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 285,
"drafted_tokens": 1416,
"accepted_tokens": 1251,
"fallback_steps": 0,
"acceptance_rate": 0.8834745763,
"acceptance_length": 5.389473684,
"accepted_per_position": [267, 258, 252, 243, 231]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 8.85e-06, "vision_seconds": 0, "prefill_seconds": 0.571859209, "decode_seconds": 4.396292774, "total_seconds": 4.968432922},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 95,
"drafted_tokens": 472,
"accepted_tokens": 417,
"fallback_steps": 0,
"acceptance_rate": 0.8834745763,
"acceptance_length": 5.389473684,
"accepted_per_position": [89, 86, 84, 81, 77]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 8.55e-06, "vision_seconds": 0, "prefill_seconds": 0.572337879, "decode_seconds": 4.396735763, "total_seconds": 4.969356152},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 95,
"drafted_tokens": 472,
"accepted_tokens": 417,
"fallback_steps": 0,
"acceptance_rate": 0.8834745763,
"acceptance_length": 5.389473684,
"accepted_per_position": [89, 86, 84, 81, 77]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 9.21e-06, "vision_seconds": 0, "prefill_seconds": 0.572653318, "decode_seconds": 4.397163703, "total_seconds": 4.970096031},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 95,
"drafted_tokens": 472,
"accepted_tokens": 417,
"fallback_steps": 0,
"acceptance_rate": 0.8834745763,
"acceptance_length": 5.389473684,
"accepted_per_position": [89, 86, 84, 81, 77]
}
}
]
},
{
"label": "pp8192+tg512",
"kind": "pp+tg",
"n_prompt": 8192,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3432.02866,
"prefill_tok_s_stddev": 1.682787014,
"decode_output_tok_s_mean": 125.793022,
"decode_output_tok_s_stddev": 0.07749953547,
"decode_engine_tok_s_mean": 125.793022,
"decode_engine_tok_s_stddev": 0.07749953547,
"prepare_seconds_mean": 2.550333333e-05,
"prepare_seconds_stddev": 2.935887146e-06,
"prefill_seconds_mean": 2.386926837,
"prefill_seconds_stddev": 0.001170184901,
"decode_seconds_mean": 4.070179105,
"decode_seconds_stddev": 0.002506856718,
"total_seconds_mean": 6.457397271,
"total_seconds_stddev": 0.002802127703,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 261,
"drafted_tokens": 1302,
"accepted_tokens": 1275,
"fallback_steps": 0,
"acceptance_rate": 0.9792626728,
"acceptance_length": 5.885057471,
"accepted_per_position": [258, 258, 255, 255, 249]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.738e-05, "vision_seconds": 0, "prefill_seconds": 2.387167149, "decode_seconds": 4.067344221, "total_seconds": 6.45479698},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 87,
"drafted_tokens": 434,
"accepted_tokens": 425,
"fallback_steps": 0,
"acceptance_rate": 0.9792626728,
"acceptance_length": 5.885057471,
"accepted_per_position": [86, 86, 85, 85, 83]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.701e-05, "vision_seconds": 0, "prefill_seconds": 2.385655151, "decode_seconds": 4.071089687, "total_seconds": 6.457029848},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 87,
"drafted_tokens": 434,
"accepted_tokens": 425,
"fallback_steps": 0,
"acceptance_rate": 0.9792626728,
"acceptance_length": 5.885057471,
"accepted_per_position": [86, 86, 85, 85, 83]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.212e-05, "vision_seconds": 0, "prefill_seconds": 2.38795821, "decode_seconds": 4.072103406, "total_seconds": 6.460364985},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 87,
"drafted_tokens": 434,
"accepted_tokens": 425,
"fallback_steps": 0,
"acceptance_rate": 0.9792626728,
"acceptance_length": 5.885057471,
"accepted_per_position": [86, 86, 85, 85, 83]
}
}
]
}
]
}