{ "schema_version": 15, "artifact_type": "ninfer_bench_report", "tool": "ninfer_bench", "command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 5 --lm-head-draft --output json --output-file /out/mtp5-gpu1-r3.json", "environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0}, "artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292}, "load": { "architecture": "Qwen3_5ForCausalLM", "name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8", "formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"], "prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc", "load_seconds": 3.361672471, "upload_seconds": 2.471311519, "artifact_bytes_read": 17501379593, "host_to_device_bytes": 17487933252, "peak_staging_bytes": 268435456, "device_object_count": 732, "host_object_count": 4 }, "memory": { "device": 0, "max_context": 16384, "kv_capacity_mode": "explicit", "kv_capacity": 16384, "kv_capacity_page_groups": 256, "kv_capacity_max_page_groups": 256, "kv_cache": "fp8-e4m3-row256", "weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064}, "sequence": {"capacity_bytes": 755319040, "used_bytes": 755319040, "peak_used_bytes": 755319040}, "workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0}, "vision_workspace": null, "minimum_runtime_reservation_bytes": 1001283840, "kv_capacity_increment_bytes": 0, "runtime_reservation_bytes": 1001283840, "available_after_weights_bytes": 7420313600, "available_after_startup_bytes": 6497566720, "kv_capacity_headroom_bytes": 0, "planned_slack_bytes": 6419029760, "cuda_graph_allowance_bytes": 85983232, "kv_payload_bytes": 575013888 }, "config": { "max_context": 16384, "prefill_chunk": 1024, "kv_cache": "fp8-e4m3-row256", "speculative_backend": "mtp", "draft_tokens": 5, "proposal_head": "optimized", "use_cuda_graph": true, "decode_path": "mtp_cuda_graph", "decode_graph_prime": {"primed": true, "output_tokens": 13}, "repetitions": 3, "warmup": 2, "corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids", "corpus_tokens": 65536 }, "tests": [ { "label": "tg256", "kind": "tg", "n_prompt": 0, "n_gen": 256, "requested_output_tokens": 257, "prefill_tok_s_mean": null, "prefill_tok_s_stddev": null, "decode_output_tok_s_mean": 45.64353647, "decode_output_tok_s_stddev": 0.01225268331, "decode_engine_tok_s_mean": 45.64353647, "decode_engine_tok_s_stddev": 0.01225268331, "prepare_seconds_mean": 1.626666667e-06, "prepare_seconds_stddev": 4.163331999e-08, "prefill_seconds_mean": null, "prefill_seconds_stddev": null, "decode_seconds_mean": 5.608680486, "decode_seconds_stddev": 0.001505812499, "total_seconds_mean": 5.646164623, "total_seconds_stddev": 0.001530725286, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 366, "drafted_tokens": 1818, "accepted_tokens": 399, "fallback_steps": 3, "acceptance_rate": 0.2194719472, "acceptance_length": 2.090163934, "accepted_per_position": [195, 111, 60, 27, 6] }, "reps": [ { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.64e-06, "vision_seconds": 0, "prefill_seconds": 0.037218125, "decode_seconds": 5.608086056, "total_seconds": 5.645572101}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 122, "drafted_tokens": 606, "accepted_tokens": 133, "fallback_steps": 1, "acceptance_rate": 0.2194719472, "acceptance_length": 2.090163934, "accepted_per_position": [65, 37, 20, 9, 2] } }, { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.58e-06, "vision_seconds": 0, "prefill_seconds": 0.037194564, "decode_seconds": 5.607562617, "total_seconds": 5.64501873}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 122, "drafted_tokens": 606, "accepted_tokens": 133, "fallback_steps": 1, "acceptance_rate": 0.2194719472, "acceptance_length": 2.090163934, "accepted_per_position": [65, 37, 20, 9, 2] } }, { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.66e-06, "vision_seconds": 0, "prefill_seconds": 0.037246264, "decode_seconds": 5.610392784, "total_seconds": 5.647903038}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 122, "drafted_tokens": 606, "accepted_tokens": 133, "fallback_steps": 1, "acceptance_rate": 0.2194719472, "acceptance_length": 2.090163934, "accepted_per_position": [65, 37, 20, 9, 2] } } ] }, { "label": "tg1024", "kind": "tg", "n_prompt": 0, "n_gen": 1024, "requested_output_tokens": 1025, "prefill_tok_s_mean": null, "prefill_tok_s_stddev": null, "decode_output_tok_s_mean": 79.96223629, "decode_output_tok_s_stddev": 0.01928508403, "decode_engine_tok_s_mean": 79.96223629, "decode_engine_tok_s_stddev": 0.01928508403, "prepare_seconds_mean": 1.86e-06, "prepare_seconds_stddev": 2.443358345e-07, "prefill_seconds_mean": null, "prefill_seconds_stddev": null, "decode_seconds_mean": 12.80604554, "decode_seconds_stddev": 0.003088215596, "total_seconds_mean": 12.84390952, "total_seconds_stddev": 0.003108343472, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 837, "drafted_tokens": 4185, "accepted_tokens": 2232, "fallback_steps": 3, "acceptance_rate": 0.5333333333, "acceptance_length": 3.666666667, "accepted_per_position": [636, 495, 405, 360, 336] }, "reps": [ { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 1.58e-06, "vision_seconds": 0, "prefill_seconds": 0.037275024, "decode_seconds": 12.80854159, "total_seconds": 12.84644666}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 279, "drafted_tokens": 1395, "accepted_tokens": 744, "fallback_steps": 1, "acceptance_rate": 0.5333333333, "acceptance_length": 3.666666667, "accepted_per_position": [212, 165, 135, 120, 112] } }, { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 1.97e-06, "vision_seconds": 0, "prefill_seconds": 0.037306394, "decode_seconds": 12.80700306, "total_seconds": 12.84483958}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 279, "drafted_tokens": 1395, "accepted_tokens": 744, "fallback_steps": 1, "acceptance_rate": 0.5333333333, "acceptance_length": 3.666666667, "accepted_per_position": [212, 165, 135, 120, 112] } }, { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 2.03e-06, "vision_seconds": 0, "prefill_seconds": 0.037319024, "decode_seconds": 12.80259198, "total_seconds": 12.84044232}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 279, "drafted_tokens": 1395, "accepted_tokens": 744, "fallback_steps": 1, "acceptance_rate": 0.5333333333, "acceptance_length": 3.666666667, "accepted_per_position": [212, 165, 135, 120, 112] } } ] }, { "label": "pp2048+tg512", "kind": "pp+tg", "n_prompt": 2048, "n_gen": 512, "requested_output_tokens": 513, "prefill_tok_s_mean": 3578.647259, "prefill_tok_s_stddev": 2.500665781, "decode_output_tok_s_mean": 116.450161, "decode_output_tok_s_stddev": 0.01153414308, "decode_engine_tok_s_mean": 116.450161, "decode_engine_tok_s_stddev": 0.01153414308, "prepare_seconds_mean": 8.87e-06, "prepare_seconds_stddev": 3.304542328e-07, "prefill_seconds_mean": 0.5722834687, "prefill_seconds_stddev": 0.0003998407674, "decode_seconds_mean": 4.396730747, "decode_seconds_stddev": 0.0004354861691, "total_seconds_mean": 4.969295035, "total_seconds_stddev": 0.0008332372725, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 285, "drafted_tokens": 1416, "accepted_tokens": 1251, "fallback_steps": 0, "acceptance_rate": 0.8834745763, "acceptance_length": 5.389473684, "accepted_per_position": [267, 258, 252, 243, 231] }, "reps": [ { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 8.85e-06, "vision_seconds": 0, "prefill_seconds": 0.571859209, "decode_seconds": 4.396292774, "total_seconds": 4.968432922}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 95, "drafted_tokens": 472, "accepted_tokens": 417, "fallback_steps": 0, "acceptance_rate": 0.8834745763, "acceptance_length": 5.389473684, "accepted_per_position": [89, 86, 84, 81, 77] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 8.55e-06, "vision_seconds": 0, "prefill_seconds": 0.572337879, "decode_seconds": 4.396735763, "total_seconds": 4.969356152}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 95, "drafted_tokens": 472, "accepted_tokens": 417, "fallback_steps": 0, "acceptance_rate": 0.8834745763, "acceptance_length": 5.389473684, "accepted_per_position": [89, 86, 84, 81, 77] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 9.21e-06, "vision_seconds": 0, "prefill_seconds": 0.572653318, "decode_seconds": 4.397163703, "total_seconds": 4.970096031}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 95, "drafted_tokens": 472, "accepted_tokens": 417, "fallback_steps": 0, "acceptance_rate": 0.8834745763, "acceptance_length": 5.389473684, "accepted_per_position": [89, 86, 84, 81, 77] } } ] }, { "label": "pp8192+tg512", "kind": "pp+tg", "n_prompt": 8192, "n_gen": 512, "requested_output_tokens": 513, "prefill_tok_s_mean": 3432.02866, "prefill_tok_s_stddev": 1.682787014, "decode_output_tok_s_mean": 125.793022, "decode_output_tok_s_stddev": 0.07749953547, "decode_engine_tok_s_mean": 125.793022, "decode_engine_tok_s_stddev": 0.07749953547, "prepare_seconds_mean": 2.550333333e-05, "prepare_seconds_stddev": 2.935887146e-06, "prefill_seconds_mean": 2.386926837, "prefill_seconds_stddev": 0.001170184901, "decode_seconds_mean": 4.070179105, "decode_seconds_stddev": 0.002506856718, "total_seconds_mean": 6.457397271, "total_seconds_stddev": 0.002802127703, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 261, "drafted_tokens": 1302, "accepted_tokens": 1275, "fallback_steps": 0, "acceptance_rate": 0.9792626728, "acceptance_length": 5.885057471, "accepted_per_position": [258, 258, 255, 255, 249] }, "reps": [ { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 2.738e-05, "vision_seconds": 0, "prefill_seconds": 2.387167149, "decode_seconds": 4.067344221, "total_seconds": 6.45479698}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 87, "drafted_tokens": 434, "accepted_tokens": 425, "fallback_steps": 0, "acceptance_rate": 0.9792626728, "acceptance_length": 5.885057471, "accepted_per_position": [86, 86, 85, 85, 83] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 2.701e-05, "vision_seconds": 0, "prefill_seconds": 2.385655151, "decode_seconds": 4.071089687, "total_seconds": 6.457029848}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 87, "drafted_tokens": 434, "accepted_tokens": 425, "fallback_steps": 0, "acceptance_rate": 0.9792626728, "acceptance_length": 5.885057471, "accepted_per_position": [86, 86, 85, 85, 83] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 2.212e-05, "vision_seconds": 0, "prefill_seconds": 2.38795821, "decode_seconds": 4.072103406, "total_seconds": 6.460364985}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 5, "rounds": 87, "drafted_tokens": 434, "accepted_tokens": 425, "fallback_steps": 0, "acceptance_rate": 0.9792626728, "acceptance_length": 5.885057471, "accepted_per_position": [86, 86, 85, 85, 83] } } ] } ] }