{ "schema_version": 15, "artifact_type": "ninfer_bench_report", "tool": "ninfer_bench", "command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 2 --lm-head-draft --output json --output-file /out/mtp2-gpu1-r3.json", "environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0}, "artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292}, "load": { "architecture": "Qwen3_5ForCausalLM", "name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8", "formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"], "prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc", "load_seconds": 3.30988622, "upload_seconds": 2.466416279, "artifact_bytes_read": 17501379593, "host_to_device_bytes": 17487933252, "peak_staging_bytes": 268435456, "device_object_count": 732, "host_object_count": 4 }, "memory": { "device": 0, "max_context": 16384, "kv_capacity_mode": "explicit", "kv_capacity": 16384, "kv_capacity_page_groups": 256, "kv_capacity_max_page_groups": 256, "kv_cache": "fp8-e4m3-row256", "weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064}, "sequence": {"capacity_bytes": 748403968, "used_bytes": 748403968, "peak_used_bytes": 748403968}, "workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0}, "vision_workspace": null, "minimum_runtime_reservation_bytes": 994368768, "kv_capacity_increment_bytes": 0, "runtime_reservation_bytes": 994368768, "available_after_weights_bytes": 7420313600, "available_after_startup_bytes": 6508052480, "kv_capacity_headroom_bytes": 0, "planned_slack_bytes": 6425944832, "cuda_graph_allowance_bytes": 85983232, "kv_payload_bytes": 575013888 }, "config": { "max_context": 16384, "prefill_chunk": 1024, "kv_cache": "fp8-e4m3-row256", "speculative_backend": "mtp", "draft_tokens": 2, "proposal_head": "optimized", "use_cuda_graph": true, "decode_path": "mtp_cuda_graph", "decode_graph_prime": {"primed": true, "output_tokens": 7}, "repetitions": 3, "warmup": 2, "corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids", "corpus_tokens": 65536 }, "tests": [ { "label": "tg256", "kind": "tg", "n_prompt": 0, "n_gen": 256, "requested_output_tokens": 257, "prefill_tok_s_mean": null, "prefill_tok_s_stddev": null, "decode_output_tok_s_mean": 54.30914011, "decode_output_tok_s_stddev": 0.03501916397, "decode_engine_tok_s_mean": 54.30914011, "decode_engine_tok_s_stddev": 0.03501916397, "prepare_seconds_mean": 1.323333333e-06, "prepare_seconds_stddev": 2.867635495e-07, "prefill_seconds_mean": null, "prefill_seconds_stddev": null, "decode_seconds_mean": 4.713756661, "decode_seconds_stddev": 0.003039106067, "total_seconds_mean": 4.7468576, "total_seconds_stddev": 0.00301945519, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 381, "drafted_tokens": 762, "accepted_tokens": 387, "fallback_steps": 0, "acceptance_rate": 0.5078740157, "acceptance_length": 2.015748031, "accepted_per_position": [240, 147] }, "reps": [ { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.02e-06, "vision_seconds": 0, "prefill_seconds": 0.03283906, "decode_seconds": 4.716577136, "total_seconds": 4.749658925}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 127, "drafted_tokens": 254, "accepted_tokens": 129, "fallback_steps": 0, "acceptance_rate": 0.5078740157, "acceptance_length": 2.015748031, "accepted_per_position": [80, 49] } }, { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.36e-06, "vision_seconds": 0, "prefill_seconds": 0.032862509, "decode_seconds": 4.710538149, "total_seconds": 4.743659288}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 127, "drafted_tokens": 254, "accepted_tokens": 129, "fallback_steps": 0, "acceptance_rate": 0.5078740157, "acceptance_length": 2.015748031, "accepted_per_position": [80, 49] } }, { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.59e-06, "vision_seconds": 0, "prefill_seconds": 0.032829549, "decode_seconds": 4.714154697, "total_seconds": 4.747254586}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 127, "drafted_tokens": 254, "accepted_tokens": 129, "fallback_steps": 0, "acceptance_rate": 0.5078740157, "acceptance_length": 2.015748031, "accepted_per_position": [80, 49] } } ] }, { "label": "tg1024", "kind": "tg", "n_prompt": 0, "n_gen": 1024, "requested_output_tokens": 1025, "prefill_tok_s_mean": null, "prefill_tok_s_stddev": null, "decode_output_tok_s_mean": 66.28016276, "decode_output_tok_s_stddev": 0.02398979789, "decode_engine_tok_s_mean": 66.28016276, "decode_engine_tok_s_stddev": 0.02398979789, "prepare_seconds_mean": 2.016666667e-06, "prepare_seconds_stddev": 5.342596123e-07, "prefill_seconds_mean": null, "prefill_seconds_stddev": null, "decode_seconds_mean": 15.44957114, "decode_seconds_stddev": 0.005592701378, "total_seconds_mean": 15.48317471, "total_seconds_stddev": 0.005541042586, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 1248, "drafted_tokens": 2493, "accepted_tokens": 1824, "fallback_steps": 0, "acceptance_rate": 0.7316486161, "acceptance_length": 2.461538462, "accepted_per_position": [1005, 819] }, "reps": [ { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 1.4e-06, "vision_seconds": 0, "prefill_seconds": 0.032878699, "decode_seconds": 15.44796308, "total_seconds": 15.48157162}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 416, "drafted_tokens": 831, "accepted_tokens": 608, "fallback_steps": 0, "acceptance_rate": 0.7316486161, "acceptance_length": 2.461538462, "accepted_per_position": [335, 273] } }, { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 2.34e-06, "vision_seconds": 0, "prefill_seconds": 0.03291765, "decode_seconds": 15.44495862, "total_seconds": 15.47861195}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 416, "drafted_tokens": 831, "accepted_tokens": 608, "fallback_steps": 0, "acceptance_rate": 0.7316486161, "acceptance_length": 2.461538462, "accepted_per_position": [335, 273] } }, { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 2.31e-06, "vision_seconds": 0, "prefill_seconds": 0.032836009, "decode_seconds": 15.45579171, "total_seconds": 15.48934055}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 416, "drafted_tokens": 831, "accepted_tokens": 608, "fallback_steps": 0, "acceptance_rate": 0.7316486161, "acceptance_length": 2.461538462, "accepted_per_position": [335, 273] } } ] }, { "label": "pp2048+tg512", "kind": "pp+tg", "n_prompt": 2048, "n_gen": 512, "requested_output_tokens": 513, "prefill_tok_s_mean": 3609.875478, "prefill_tok_s_stddev": 3.395740868, "decode_output_tok_s_mean": 77.14063636, "decode_output_tok_s_stddev": 0.01793529499, "decode_engine_tok_s_mean": 77.14063636, "decode_engine_tok_s_stddev": 0.01793529499, "prepare_seconds_mean": 9.503333333e-06, "prepare_seconds_stddev": 2.34141695e-06, "prefill_seconds_mean": 0.5673329233, "prefill_seconds_stddev": 0.0005336321206, "decode_seconds_mean": 6.637228349, "decode_seconds_stddev": 0.001543341519, "total_seconds_mean": 7.204939955, "total_seconds_stddev": 0.001964671337, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 531, "drafted_tokens": 1062, "accepted_tokens": 1005, "fallback_steps": 0, "acceptance_rate": 0.9463276836, "acceptance_length": 2.892655367, "accepted_per_position": [513, 492] }, "reps": [ { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 6.8e-06, "vision_seconds": 0, "prefill_seconds": 0.567366283, "decode_seconds": 6.636074959, "total_seconds": 7.203827342}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 177, "drafted_tokens": 354, "accepted_tokens": 335, "fallback_steps": 0, "acceptance_rate": 0.9463276836, "acceptance_length": 2.892655367, "accepted_per_position": [171, 164] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 1.082e-05, "vision_seconds": 0, "prefill_seconds": 0.567849093, "decode_seconds": 6.638981552, "total_seconds": 7.207208425}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 177, "drafted_tokens": 354, "accepted_tokens": 335, "fallback_steps": 0, "acceptance_rate": 0.9463276836, "acceptance_length": 2.892655367, "accepted_per_position": [171, 164] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 1.089e-05, "vision_seconds": 0, "prefill_seconds": 0.566783394, "decode_seconds": 6.636628535, "total_seconds": 7.203784099}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 177, "drafted_tokens": 354, "accepted_tokens": 335, "fallback_steps": 0, "acceptance_rate": 0.9463276836, "acceptance_length": 2.892655367, "accepted_per_position": [171, 164] } } ] }, { "label": "pp8192+tg512", "kind": "pp+tg", "n_prompt": 8192, "n_gen": 512, "requested_output_tokens": 513, "prefill_tok_s_mean": 3432.52722, "prefill_tok_s_stddev": 2.260810446, "decode_output_tok_s_mean": 78.41489219, "decode_output_tok_s_stddev": 0.003160414981, "decode_engine_tok_s_mean": 78.41489219, "decode_engine_tok_s_stddev": 0.003160414981, "prepare_seconds_mean": 2.975e-05, "prepare_seconds_stddev": 1.68e-06, "prefill_seconds_mean": 2.386580454, "prefill_seconds_stddev": 0.001571975591, "decode_seconds_mean": 6.529371988, "decode_seconds_stddev": 0.0002631612017, "total_seconds_mean": 8.916357365, "total_seconds_stddev": 0.001828984428, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 516, "drafted_tokens": 1032, "accepted_tokens": 1020, "fallback_steps": 0, "acceptance_rate": 0.988372093, "acceptance_length": 2.976744186, "accepted_per_position": [510, 510] }, "reps": [ { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 3.143e-05, "vision_seconds": 0, "prefill_seconds": 2.388187259, "decode_seconds": 6.529656878, "total_seconds": 8.918243587}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 172, "drafted_tokens": 344, "accepted_tokens": 340, "fallback_steps": 0, "acceptance_rate": 0.988372093, "acceptance_length": 2.976744186, "accepted_per_position": [170, 170] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 2.807e-05, "vision_seconds": 0, "prefill_seconds": 2.38650831, "decode_seconds": 6.529321098, "total_seconds": 8.916236938}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 172, "drafted_tokens": 344, "accepted_tokens": 340, "fallback_steps": 0, "acceptance_rate": 0.988372093, "acceptance_length": 2.976744186, "accepted_per_position": [170, 170] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 2.975e-05, "vision_seconds": 0, "prefill_seconds": 2.385045792, "decode_seconds": 6.529137989, "total_seconds": 8.91459157}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 2, "rounds": 172, "drafted_tokens": 344, "accepted_tokens": 340, "fallback_steps": 0, "acceptance_rate": 0.988372093, "acceptance_length": 2.976744186, "accepted_per_position": [170, 170] } } ] } ] }