Hugging Face's logo

Schestex
/
ThinkingCap-Qwen3.8-27B-NInfer

Text Generation
NInfer
qwen.83
qwen3_8
blackwell
sm120a
mtp
speculative-decoding
indexed88
vocabulary-prune
coding
agentic
nvfp4
conversational
Model card Files
xet
 Community
5
ThinkingCap-Qwen3.8-27B-NInfer / bench /mtp3-gpu1-145w-r3.json
Schestex's picture
Schestex
RC1: D48+O44+I53+G64 + native Q8 MTP + Sharp
b78a22f verified
Raw History Blame Contribute Delete
16.3 kB
{
"schema_version": 15,
"artifact_type": "ninfer_bench_report",
"tool": "ninfer_bench",
"command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 3 --lm-head-draft --output json --output-file /out/mtp3-gpu1-r3.json",
"environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0},
"artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292},
"load": {
"architecture": "Qwen3_5ForCausalLM",
"name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8",
"formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"],
"prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc",
"load_seconds": 3.316723434,
"upload_seconds": 2.466533213,
"artifact_bytes_read": 17501379593,
"host_to_device_bytes": 17487933252,
"peak_staging_bytes": 268435456,
"device_object_count": 732,
"host_object_count": 4
},
"memory": {
"device": 0,
"max_context": 16384,
"kv_capacity_mode": "explicit",
"kv_capacity": 16384,
"kv_capacity_page_groups": 256,
"kv_capacity_max_page_groups": 256,
"kv_cache": "fp8-e4m3-row256",
"weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064},
"sequence": {"capacity_bytes": 750708992, "used_bytes": 750708992, "peak_used_bytes": 750708992},
"workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0},
"vision_workspace": null,
"minimum_runtime_reservation_bytes": 996673792,
"kv_capacity_increment_bytes": 0,
"runtime_reservation_bytes": 996673792,
"available_after_weights_bytes": 7420313600,
"available_after_startup_bytes": 6505955328,
"kv_capacity_headroom_bytes": 0,
"planned_slack_bytes": 6423639808,
"cuda_graph_allowance_bytes": 85983232,
"kv_payload_bytes": 575013888
},
"config": {
"max_context": 16384,
"prefill_chunk": 1024,
"kv_cache": "fp8-e4m3-row256",
"speculative_backend": "mtp",
"draft_tokens": 3,
"proposal_head": "optimized",
"use_cuda_graph": true,
"decode_path": "mtp_cuda_graph",
"decode_graph_prime": {"primed": true, "output_tokens": 9},
"repetitions": 3,
"warmup": 2,
"corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids",
"corpus_tokens": 65536
},
"tests": [
{
"label": "tg256",
"kind": "tg",
"n_prompt": 0,
"n_gen": 256,
"requested_output_tokens": 257,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 54.16346999,
"decode_output_tok_s_stddev": 0.02061488267,
"decode_engine_tok_s_mean": 54.16346999,
"decode_engine_tok_s_stddev": 0.02061488267,
"prepare_seconds_mean": 1.31e-06,
"prepare_seconds_stddev": 2.351595203e-07,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 4.726433236,
"decode_seconds_stddev": 0.00179857864,
"total_seconds_mean": 4.760920559,
"total_seconds_stddev": 0.001817732811,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 342,
"drafted_tokens": 1023,
"accepted_tokens": 426,
"fallback_steps": 0,
"acceptance_rate": 0.4164222874,
"acceptance_length": 2.245614035,
"accepted_per_position": [219, 141, 66]
},
"reps": [
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.3e-06, "vision_seconds": 0, "prefill_seconds": 0.034216187, "decode_seconds": 4.724398538, "total_seconds": 4.758862795},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 114,
"drafted_tokens": 341,
"accepted_tokens": 142,
"fallback_steps": 0,
"acceptance_rate": 0.4164222874,
"acceptance_length": 2.245614035,
"accepted_per_position": [73, 47, 22]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.08e-06, "vision_seconds": 0, "prefill_seconds": 0.034256036, "decode_seconds": 4.727810985, "total_seconds": 4.762307711},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 114,
"drafted_tokens": 341,
"accepted_tokens": 142,
"fallback_steps": 0,
"acceptance_rate": 0.4164222874,
"acceptance_length": 2.245614035,
"accepted_per_position": [73, 47, 22]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.55e-06, "vision_seconds": 0, "prefill_seconds": 0.034265667, "decode_seconds": 4.727090185, "total_seconds": 4.761591172},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 114,
"drafted_tokens": 341,
"accepted_tokens": 142,
"fallback_steps": 0,
"acceptance_rate": 0.4164222874,
"acceptance_length": 2.245614035,
"accepted_per_position": [73, 47, 22]
}
}
]
},
{
"label": "tg1024",
"kind": "tg",
"n_prompt": 0,
"n_gen": 1024,
"requested_output_tokens": 1025,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 58.0845176,
"decode_output_tok_s_stddev": 0.00498641247,
"decode_engine_tok_s_mean": 58.0845176,
"decode_engine_tok_s_stddev": 0.00498641247,
"prepare_seconds_mean": 1.92e-06,
"prepare_seconds_stddev": 6.850547423e-07,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 17.62948282,
"decode_seconds_stddev": 0.001513520352,
"total_seconds_mean": 17.66459143,
"total_seconds_stddev": 0.001473746939,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 1269,
"drafted_tokens": 3804,
"accepted_tokens": 1800,
"fallback_steps": 3,
"acceptance_rate": 0.4731861199,
"acceptance_length": 2.418439716,
"accepted_per_position": [909, 576, 315]
},
"reps": [
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 1.13e-06, "vision_seconds": 0, "prefill_seconds": 0.034378477, "decode_seconds": 17.62873566, "total_seconds": 17.66394648},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 423,
"drafted_tokens": 1268,
"accepted_tokens": 600,
"fallback_steps": 1,
"acceptance_rate": 0.4731861199,
"acceptance_length": 2.418439716,
"accepted_per_position": [303, 192, 105]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 2.28e-06, "vision_seconds": 0, "prefill_seconds": 0.034313017, "decode_seconds": 17.63122463, "total_seconds": 17.66627771},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 423,
"drafted_tokens": 1268,
"accepted_tokens": 600,
"fallback_steps": 1,
"acceptance_rate": 0.4731861199,
"acceptance_length": 2.418439716,
"accepted_per_position": [303, 192, 105]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 2.35e-06, "vision_seconds": 0, "prefill_seconds": 0.034337167, "decode_seconds": 17.62848816, "total_seconds": 17.6635501},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 423,
"drafted_tokens": 1268,
"accepted_tokens": 600,
"fallback_steps": 1,
"acceptance_rate": 0.4731861199,
"acceptance_length": 2.418439716,
"accepted_per_position": [303, 192, 105]
}
}
]
},
{
"label": "pp2048+tg512",
"kind": "pp+tg",
"n_prompt": 2048,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3599.547483,
"prefill_tok_s_stddev": 2.252708112,
"decode_output_tok_s_mean": 89.6428123,
"decode_output_tok_s_stddev": 0.02176244083,
"decode_engine_tok_s_mean": 89.6428123,
"decode_engine_tok_s_stddev": 0.02176244083,
"prepare_seconds_mean": 9.456666667e-06,
"prepare_seconds_stddev": 1.209021643e-06,
"prefill_seconds_mean": 0.5689605553,
"prefill_seconds_stddev": 0.0003559983107,
"decode_seconds_mean": 5.711556866,
"decode_seconds_stddev": 0.00138663725,
"total_seconds_mean": 6.280836125,
"total_seconds_stddev": 0.001156129626,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 408,
"drafted_tokens": 1221,
"accepted_tokens": 1128,
"fallback_steps": 0,
"acceptance_rate": 0.9238329238,
"acceptance_length": 3.764705882,
"accepted_per_position": [384, 378, 366]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 1.029e-05, "vision_seconds": 0, "prefill_seconds": 0.568569952, "decode_seconds": 5.713009855, "total_seconds": 6.281900687},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 136,
"drafted_tokens": 407,
"accepted_tokens": 376,
"fallback_steps": 0,
"acceptance_rate": 0.9238329238,
"acceptance_length": 3.764705882,
"accepted_per_position": [128, 126, 122]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 8.07e-06, "vision_seconds": 0, "prefill_seconds": 0.569044922, "decode_seconds": 5.710247808, "total_seconds": 6.279606229},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 136,
"drafted_tokens": 407,
"accepted_tokens": 376,
"fallback_steps": 0,
"acceptance_rate": 0.9238329238,
"acceptance_length": 3.764705882,
"accepted_per_position": [128, 126, 122]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 1.001e-05, "vision_seconds": 0, "prefill_seconds": 0.569266792, "decode_seconds": 5.711412936, "total_seconds": 6.281001458},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 136,
"drafted_tokens": 407,
"accepted_tokens": 376,
"fallback_steps": 0,
"acceptance_rate": 0.9238329238,
"acceptance_length": 3.764705882,
"accepted_per_position": [128, 126, 122]
}
}
]
},
{
"label": "pp8192+tg512",
"kind": "pp+tg",
"n_prompt": 8192,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3436.856537,
"prefill_tok_s_stddev": 0.2878826734,
"decode_output_tok_s_mean": 93.5343478,
"decode_output_tok_s_stddev": 0.02561220332,
"decode_engine_tok_s_mean": 93.5343478,
"decode_engine_tok_s_stddev": 0.02561220332,
"prepare_seconds_mean": 2.846666667e-05,
"prepare_seconds_stddev": 1.435142269e-06,
"prefill_seconds_mean": 2.383573463,
"prefill_seconds_stddev": 0.0001996630351,
"decode_seconds_mean": 5.473925222,
"decode_seconds_stddev": 0.001498824652,
"total_seconds_mean": 7.857857098,
"total_seconds_stddev": 0.001348104877,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 387,
"drafted_tokens": 1161,
"accepted_tokens": 1149,
"fallback_steps": 0,
"acceptance_rate": 0.9896640827,
"acceptance_length": 3.968992248,
"accepted_per_position": [387, 384, 378]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.783e-05, "vision_seconds": 0, "prefill_seconds": 2.383513023, "decode_seconds": 5.475311554, "total_seconds": 7.859187347},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 129,
"drafted_tokens": 387,
"accepted_tokens": 383,
"fallback_steps": 0,
"acceptance_rate": 0.9896640827,
"acceptance_length": 3.968992248,
"accepted_per_position": [129, 128, 126]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 3.011e-05, "vision_seconds": 0, "prefill_seconds": 2.383411003, "decode_seconds": 5.474129296, "total_seconds": 7.857892128},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 129,
"drafted_tokens": 387,
"accepted_tokens": 383,
"fallback_steps": 0,
"acceptance_rate": 0.9896640827,
"acceptance_length": 3.968992248,
"accepted_per_position": [129, 128, 126]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.746e-05, "vision_seconds": 0, "prefill_seconds": 2.383796363, "decode_seconds": 5.472334817, "total_seconds": 7.85649182},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 129,
"drafted_tokens": 387,
"accepted_tokens": 383,
"fallback_steps": 0,
"acceptance_rate": 0.9896640827,
"acceptance_length": 3.968992248,
"accepted_per_position": [129, 128, 126]
}
}
]
}
]
}