{ "schema_version": 15, "artifact_type": "ninfer_bench_report", "tool": "ninfer_bench", "command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 4 --lm-head-draft --output json --output-file /out/mtp4-gpu1-r3.json", "environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0}, "artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292}, "load": { "architecture": "Qwen3_5ForCausalLM", "name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8", "formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"], "prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc", "load_seconds": 3.31989257, "upload_seconds": 2.462935676, "artifact_bytes_read": 17501379593, "host_to_device_bytes": 17487933252, "peak_staging_bytes": 268435456, "device_object_count": 732, "host_object_count": 4 }, "memory": { "device": 0, "max_context": 16384, "kv_capacity_mode": "explicit", "kv_capacity": 16384, "kv_capacity_page_groups": 256, "kv_capacity_max_page_groups": 256, "kv_cache": "fp8-e4m3-row256", "weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064}, "sequence": {"capacity_bytes": 753014016, "used_bytes": 753014016, "peak_used_bytes": 753014016}, "workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0}, "vision_workspace": null, "minimum_runtime_reservation_bytes": 998978816, "kv_capacity_increment_bytes": 0, "runtime_reservation_bytes": 998978816, "available_after_weights_bytes": 7420313600, "available_after_startup_bytes": 6499663872, "kv_capacity_headroom_bytes": 0, "planned_slack_bytes": 6421334784, "cuda_graph_allowance_bytes": 85983232, "kv_payload_bytes": 575013888 }, "config": { "max_context": 16384, "prefill_chunk": 1024, "kv_cache": "fp8-e4m3-row256", "speculative_backend": "mtp", "draft_tokens": 4, "proposal_head": "optimized", "use_cuda_graph": true, "decode_path": "mtp_cuda_graph", "decode_graph_prime": {"primed": true, "output_tokens": 11}, "repetitions": 3, "warmup": 2, "corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids", "corpus_tokens": 65536 }, "tests": [ { "label": "tg256", "kind": "tg", "n_prompt": 0, "n_gen": 256, "requested_output_tokens": 257, "prefill_tok_s_mean": null, "prefill_tok_s_stddev": null, "decode_output_tok_s_mean": 50.40448937, "decode_output_tok_s_stddev": 0.03083291091, "decode_engine_tok_s_mean": 50.40448937, "decode_engine_tok_s_stddev": 0.03083291091, "prepare_seconds_mean": 1.626666667e-06, "prepare_seconds_stddev": 1.553490693e-07, "prefill_seconds_mean": null, "prefill_seconds_stddev": null, "decode_seconds_mean": 5.078913943, "decode_seconds_stddev": 0.00310731929, "total_seconds_mean": 5.114882921, "total_seconds_stddev": 0.003115239368, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 360, "drafted_tokens": 1434, "accepted_tokens": 405, "fallback_steps": 3, "acceptance_rate": 0.2824267782, "acceptance_length": 2.125, "accepted_per_position": [198, 114, 66, 27] }, "reps": [ { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.5e-06, "vision_seconds": 0, "prefill_seconds": 0.035708685, "decode_seconds": 5.076126319, "total_seconds": 5.112076764}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 120, "drafted_tokens": 478, "accepted_tokens": 135, "fallback_steps": 1, "acceptance_rate": 0.2824267782, "acceptance_length": 2.125, "accepted_per_position": [66, 38, 22, 9] } }, { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.58e-06, "vision_seconds": 0, "prefill_seconds": 0.035758895, "decode_seconds": 5.078351407, "total_seconds": 5.114336942}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 120, "drafted_tokens": 478, "accepted_tokens": 135, "fallback_steps": 1, "acceptance_rate": 0.2824267782, "acceptance_length": 2.125, "accepted_per_position": [66, 38, 22, 9] } }, { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.8e-06, "vision_seconds": 0, "prefill_seconds": 0.035727875, "decode_seconds": 5.082264103, "total_seconds": 5.118235058}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 120, "drafted_tokens": 478, "accepted_tokens": 135, "fallback_steps": 1, "acceptance_rate": 0.2824267782, "acceptance_length": 2.125, "accepted_per_position": [66, 38, 22, 9] } } ] }, { "label": "tg1024", "kind": "tg", "n_prompt": 0, "n_gen": 1024, "requested_output_tokens": 1025, "prefill_tok_s_mean": null, "prefill_tok_s_stddev": null, "decode_output_tok_s_mean": 78.21586299, "decode_output_tok_s_stddev": 0.01818379148, "decode_engine_tok_s_mean": 78.21586299, "decode_engine_tok_s_stddev": 0.01818379148, "prepare_seconds_mean": 2.146666667e-06, "prepare_seconds_stddev": 7.805340052e-07, "prefill_seconds_mean": null, "prefill_seconds_stddev": null, "decode_seconds_mean": 13.0919739, "decode_seconds_stddev": 0.003043243019, "total_seconds_mean": 13.12834902, "total_seconds_stddev": 0.003057889399, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 933, "drafted_tokens": 3726, "accepted_tokens": 2139, "fallback_steps": 0, "acceptance_rate": 0.5740740741, "acceptance_length": 3.292604502, "accepted_per_position": [708, 573, 468, 390] }, "reps": [ { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 2.18e-06, "vision_seconds": 0, "prefill_seconds": 0.035744135, "decode_seconds": 13.08846115, "total_seconds": 13.12482131}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 311, "drafted_tokens": 1242, "accepted_tokens": 713, "fallback_steps": 0, "acceptance_rate": 0.5740740741, "acceptance_length": 3.292604502, "accepted_per_position": [236, 191, 156, 130] } }, { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 1.35e-06, "vision_seconds": 0, "prefill_seconds": 0.035847435, "decode_seconds": 13.09381244, "total_seconds": 13.13024384}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 311, "drafted_tokens": 1242, "accepted_tokens": 713, "fallback_steps": 0, "acceptance_rate": 0.5740740741, "acceptance_length": 3.292604502, "accepted_per_position": [236, 191, 156, 130] } }, { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 2.91e-06, "vision_seconds": 0, "prefill_seconds": 0.035725165, "decode_seconds": 13.09364811, "total_seconds": 13.12998191}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 311, "drafted_tokens": 1242, "accepted_tokens": 713, "fallback_steps": 0, "acceptance_rate": 0.5740740741, "acceptance_length": 3.292604502, "accepted_per_position": [236, 191, 156, 130] } } ] }, { "label": "pp2048+tg512", "kind": "pp+tg", "n_prompt": 2048, "n_gen": 512, "requested_output_tokens": 513, "prefill_tok_s_mean": 3595.045978, "prefill_tok_s_stddev": 1.155058709, "decode_output_tok_s_mean": 108.3453813, "decode_output_tok_s_stddev": 0.01947590132, "decode_engine_tok_s_mean": 108.3453813, "decode_engine_tok_s_stddev": 0.01947590132, "prepare_seconds_mean": 9.63e-06, "prepare_seconds_stddev": 7.35119038e-07, "prefill_seconds_mean": 0.5696728647, "prefill_seconds_stddev": 0.000183034102, "decode_seconds_mean": 4.725628399, "decode_seconds_stddev": 0.0008494409399, "total_seconds_mean": 5.295581607, "total_seconds_stddev": 0.0007932659679, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 333, "drafted_tokens": 1332, "accepted_tokens": 1203, "fallback_steps": 0, "acceptance_rate": 0.9031531532, "acceptance_length": 4.612612613, "accepted_per_position": [321, 309, 297, 276] }, "reps": [ { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 1.045e-05, "vision_seconds": 0, "prefill_seconds": 0.569492922, "decode_seconds": 4.726423675, "total_seconds": 5.296199627}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 111, "drafted_tokens": 444, "accepted_tokens": 401, "fallback_steps": 0, "acceptance_rate": 0.9031531532, "acceptance_length": 4.612612613, "accepted_per_position": [107, 103, 99, 92] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 9.41e-06, "vision_seconds": 0, "prefill_seconds": 0.569858841, "decode_seconds": 4.725727956, "total_seconds": 5.295858097}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 111, "drafted_tokens": 444, "accepted_tokens": 401, "fallback_steps": 0, "acceptance_rate": 0.9031531532, "acceptance_length": 4.612612613, "accepted_per_position": [107, 103, 99, 92] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 9.03e-06, "vision_seconds": 0, "prefill_seconds": 0.569666831, "decode_seconds": 4.724733567, "total_seconds": 5.294687098}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 111, "drafted_tokens": 444, "accepted_tokens": 401, "fallback_steps": 0, "acceptance_rate": 0.9031531532, "acceptance_length": 4.612612613, "accepted_per_position": [107, 103, 99, 92] } } ] }, { "label": "pp8192+tg512", "kind": "pp+tg", "n_prompt": 8192, "n_gen": 512, "requested_output_tokens": 513, "prefill_tok_s_mean": 3437.138454, "prefill_tok_s_stddev": 1.0925128, "decode_output_tok_s_mean": 114.3989504, "decode_output_tok_s_stddev": 0.03672599078, "decode_engine_tok_s_mean": 114.3989504, "decode_engine_tok_s_stddev": 0.03672599078, "prepare_seconds_mean": 3.007666667e-05, "prepare_seconds_stddev": 6.476367295e-07, "prefill_seconds_mean": 2.38337811, "prefill_seconds_stddev": 0.0007574844131, "decode_seconds_mean": 4.475565844, "decode_seconds_stddev": 0.001437074982, "total_seconds_mean": 6.85935043, "total_seconds_stddev": 0.001811280528, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 312, "drafted_tokens": 1242, "accepted_tokens": 1224, "fallback_steps": 0, "acceptance_rate": 0.9855072464, "acceptance_length": 4.923076923, "accepted_per_position": [309, 309, 303, 303] }, "reps": [ { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 2.936e-05, "vision_seconds": 0, "prefill_seconds": 2.384022283, "decode_seconds": 4.474790459, "total_seconds": 6.859203871}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 104, "drafted_tokens": 414, "accepted_tokens": 408, "fallback_steps": 0, "acceptance_rate": 0.9855072464, "acceptance_length": 4.923076923, "accepted_per_position": [103, 103, 101, 101] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 3.062e-05, "vision_seconds": 0, "prefill_seconds": 2.382543614, "decode_seconds": 4.474682998, "total_seconds": 6.857616882}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 104, "drafted_tokens": 414, "accepted_tokens": 408, "fallback_steps": 0, "acceptance_rate": 0.9855072464, "acceptance_length": 4.923076923, "accepted_per_position": [103, 103, 101, 101] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 3.025e-05, "vision_seconds": 0, "prefill_seconds": 2.383568433, "decode_seconds": 4.477224075, "total_seconds": 6.861230538}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 4, "rounds": 104, "drafted_tokens": 414, "accepted_tokens": 408, "fallback_steps": 0, "acceptance_rate": 0.9855072464, "acceptance_length": 4.923076923, "accepted_per_position": [103, 103, 101, 101] } } ] } ] }