{ "schema_version": 15, "artifact_type": "ninfer_bench_report", "tool": "ninfer_bench", "command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 3 --lm-head-draft --output json --output-file /out/mtp3-gpu1-r3.json", "environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0}, "artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292}, "load": { "architecture": "Qwen3_5ForCausalLM", "name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8", "formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"], "prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc", "load_seconds": 3.316723434, "upload_seconds": 2.466533213, "artifact_bytes_read": 17501379593, "host_to_device_bytes": 17487933252, "peak_staging_bytes": 268435456, "device_object_count": 732, "host_object_count": 4 }, "memory": { "device": 0, "max_context": 16384, "kv_capacity_mode": "explicit", "kv_capacity": 16384, "kv_capacity_page_groups": 256, "kv_capacity_max_page_groups": 256, "kv_cache": "fp8-e4m3-row256", "weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064}, "sequence": {"capacity_bytes": 750708992, "used_bytes": 750708992, "peak_used_bytes": 750708992}, "workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0}, "vision_workspace": null, "minimum_runtime_reservation_bytes": 996673792, "kv_capacity_increment_bytes": 0, "runtime_reservation_bytes": 996673792, "available_after_weights_bytes": 7420313600, "available_after_startup_bytes": 6505955328, "kv_capacity_headroom_bytes": 0, "planned_slack_bytes": 6423639808, "cuda_graph_allowance_bytes": 85983232, "kv_payload_bytes": 575013888 }, "config": { "max_context": 16384, "prefill_chunk": 1024, "kv_cache": "fp8-e4m3-row256", "speculative_backend": "mtp", "draft_tokens": 3, "proposal_head": "optimized", "use_cuda_graph": true, "decode_path": "mtp_cuda_graph", "decode_graph_prime": {"primed": true, "output_tokens": 9}, "repetitions": 3, "warmup": 2, "corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids", "corpus_tokens": 65536 }, "tests": [ { "label": "tg256", "kind": "tg", "n_prompt": 0, "n_gen": 256, "requested_output_tokens": 257, "prefill_tok_s_mean": null, "prefill_tok_s_stddev": null, "decode_output_tok_s_mean": 54.16346999, "decode_output_tok_s_stddev": 0.02061488267, "decode_engine_tok_s_mean": 54.16346999, "decode_engine_tok_s_stddev": 0.02061488267, "prepare_seconds_mean": 1.31e-06, "prepare_seconds_stddev": 2.351595203e-07, "prefill_seconds_mean": null, "prefill_seconds_stddev": null, "decode_seconds_mean": 4.726433236, "decode_seconds_stddev": 0.00179857864, "total_seconds_mean": 4.760920559, "total_seconds_stddev": 0.001817732811, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 342, "drafted_tokens": 1023, "accepted_tokens": 426, "fallback_steps": 0, "acceptance_rate": 0.4164222874, "acceptance_length": 2.245614035, "accepted_per_position": [219, 141, 66] }, "reps": [ { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.3e-06, "vision_seconds": 0, "prefill_seconds": 0.034216187, "decode_seconds": 4.724398538, "total_seconds": 4.758862795}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 114, "drafted_tokens": 341, "accepted_tokens": 142, "fallback_steps": 0, "acceptance_rate": 0.4164222874, "acceptance_length": 2.245614035, "accepted_per_position": [73, 47, 22] } }, { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.08e-06, "vision_seconds": 0, "prefill_seconds": 0.034256036, "decode_seconds": 4.727810985, "total_seconds": 4.762307711}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 114, "drafted_tokens": 341, "accepted_tokens": 142, "fallback_steps": 0, "acceptance_rate": 0.4164222874, "acceptance_length": 2.245614035, "accepted_per_position": [73, 47, 22] } }, { "generated_output_tokens": 257, "decode_output_tokens": 256, "decode_engine_tokens": 256, "timings": {"prepare_seconds": 1.55e-06, "vision_seconds": 0, "prefill_seconds": 0.034265667, "decode_seconds": 4.727090185, "total_seconds": 4.761591172}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 114, "drafted_tokens": 341, "accepted_tokens": 142, "fallback_steps": 0, "acceptance_rate": 0.4164222874, "acceptance_length": 2.245614035, "accepted_per_position": [73, 47, 22] } } ] }, { "label": "tg1024", "kind": "tg", "n_prompt": 0, "n_gen": 1024, "requested_output_tokens": 1025, "prefill_tok_s_mean": null, "prefill_tok_s_stddev": null, "decode_output_tok_s_mean": 58.0845176, "decode_output_tok_s_stddev": 0.00498641247, "decode_engine_tok_s_mean": 58.0845176, "decode_engine_tok_s_stddev": 0.00498641247, "prepare_seconds_mean": 1.92e-06, "prepare_seconds_stddev": 6.850547423e-07, "prefill_seconds_mean": null, "prefill_seconds_stddev": null, "decode_seconds_mean": 17.62948282, "decode_seconds_stddev": 0.001513520352, "total_seconds_mean": 17.66459143, "total_seconds_stddev": 0.001473746939, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 1269, "drafted_tokens": 3804, "accepted_tokens": 1800, "fallback_steps": 3, "acceptance_rate": 0.4731861199, "acceptance_length": 2.418439716, "accepted_per_position": [909, 576, 315] }, "reps": [ { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 1.13e-06, "vision_seconds": 0, "prefill_seconds": 0.034378477, "decode_seconds": 17.62873566, "total_seconds": 17.66394648}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 423, "drafted_tokens": 1268, "accepted_tokens": 600, "fallback_steps": 1, "acceptance_rate": 0.4731861199, "acceptance_length": 2.418439716, "accepted_per_position": [303, 192, 105] } }, { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 2.28e-06, "vision_seconds": 0, "prefill_seconds": 0.034313017, "decode_seconds": 17.63122463, "total_seconds": 17.66627771}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 423, "drafted_tokens": 1268, "accepted_tokens": 600, "fallback_steps": 1, "acceptance_rate": 0.4731861199, "acceptance_length": 2.418439716, "accepted_per_position": [303, 192, 105] } }, { "generated_output_tokens": 1025, "decode_output_tokens": 1024, "decode_engine_tokens": 1024, "timings": {"prepare_seconds": 2.35e-06, "vision_seconds": 0, "prefill_seconds": 0.034337167, "decode_seconds": 17.62848816, "total_seconds": 17.6635501}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 423, "drafted_tokens": 1268, "accepted_tokens": 600, "fallback_steps": 1, "acceptance_rate": 0.4731861199, "acceptance_length": 2.418439716, "accepted_per_position": [303, 192, 105] } } ] }, { "label": "pp2048+tg512", "kind": "pp+tg", "n_prompt": 2048, "n_gen": 512, "requested_output_tokens": 513, "prefill_tok_s_mean": 3599.547483, "prefill_tok_s_stddev": 2.252708112, "decode_output_tok_s_mean": 89.6428123, "decode_output_tok_s_stddev": 0.02176244083, "decode_engine_tok_s_mean": 89.6428123, "decode_engine_tok_s_stddev": 0.02176244083, "prepare_seconds_mean": 9.456666667e-06, "prepare_seconds_stddev": 1.209021643e-06, "prefill_seconds_mean": 0.5689605553, "prefill_seconds_stddev": 0.0003559983107, "decode_seconds_mean": 5.711556866, "decode_seconds_stddev": 0.00138663725, "total_seconds_mean": 6.280836125, "total_seconds_stddev": 0.001156129626, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 408, "drafted_tokens": 1221, "accepted_tokens": 1128, "fallback_steps": 0, "acceptance_rate": 0.9238329238, "acceptance_length": 3.764705882, "accepted_per_position": [384, 378, 366] }, "reps": [ { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 1.029e-05, "vision_seconds": 0, "prefill_seconds": 0.568569952, "decode_seconds": 5.713009855, "total_seconds": 6.281900687}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 136, "drafted_tokens": 407, "accepted_tokens": 376, "fallback_steps": 0, "acceptance_rate": 0.9238329238, "acceptance_length": 3.764705882, "accepted_per_position": [128, 126, 122] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 8.07e-06, "vision_seconds": 0, "prefill_seconds": 0.569044922, "decode_seconds": 5.710247808, "total_seconds": 6.279606229}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 136, "drafted_tokens": 407, "accepted_tokens": 376, "fallback_steps": 0, "acceptance_rate": 0.9238329238, "acceptance_length": 3.764705882, "accepted_per_position": [128, 126, 122] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 1.001e-05, "vision_seconds": 0, "prefill_seconds": 0.569266792, "decode_seconds": 5.711412936, "total_seconds": 6.281001458}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 136, "drafted_tokens": 407, "accepted_tokens": 376, "fallback_steps": 0, "acceptance_rate": 0.9238329238, "acceptance_length": 3.764705882, "accepted_per_position": [128, 126, 122] } } ] }, { "label": "pp8192+tg512", "kind": "pp+tg", "n_prompt": 8192, "n_gen": 512, "requested_output_tokens": 513, "prefill_tok_s_mean": 3436.856537, "prefill_tok_s_stddev": 0.2878826734, "decode_output_tok_s_mean": 93.5343478, "decode_output_tok_s_stddev": 0.02561220332, "decode_engine_tok_s_mean": 93.5343478, "decode_engine_tok_s_stddev": 0.02561220332, "prepare_seconds_mean": 2.846666667e-05, "prepare_seconds_stddev": 1.435142269e-06, "prefill_seconds_mean": 2.383573463, "prefill_seconds_stddev": 0.0001996630351, "decode_seconds_mean": 5.473925222, "decode_seconds_stddev": 0.001498824652, "total_seconds_mean": 7.857857098, "total_seconds_stddev": 0.001348104877, "workspace_peak_bytes": 159981568, "workspace_allocator_peak_bytes": 159981568, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 387, "drafted_tokens": 1161, "accepted_tokens": 1149, "fallback_steps": 0, "acceptance_rate": 0.9896640827, "acceptance_length": 3.968992248, "accepted_per_position": [387, 384, 378] }, "reps": [ { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 2.783e-05, "vision_seconds": 0, "prefill_seconds": 2.383513023, "decode_seconds": 5.475311554, "total_seconds": 7.859187347}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 129, "drafted_tokens": 387, "accepted_tokens": 383, "fallback_steps": 0, "acceptance_rate": 0.9896640827, "acceptance_length": 3.968992248, "accepted_per_position": [129, 128, 126] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 3.011e-05, "vision_seconds": 0, "prefill_seconds": 2.383411003, "decode_seconds": 5.474129296, "total_seconds": 7.857892128}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 129, "drafted_tokens": 387, "accepted_tokens": 383, "fallback_steps": 0, "acceptance_rate": 0.9896640827, "acceptance_length": 3.968992248, "accepted_per_position": [129, 128, 126] } }, { "generated_output_tokens": 513, "decode_output_tokens": 512, "decode_engine_tokens": 512, "timings": {"prepare_seconds": 2.746e-05, "vision_seconds": 0, "prefill_seconds": 2.383796363, "decode_seconds": 5.472334817, "total_seconds": 7.85649182}, "speculative": { "backend": "mtp", "enabled": true, "draft_window": 3, "rounds": 129, "drafted_tokens": 387, "accepted_tokens": 383, "fallback_steps": 0, "acceptance_rate": 0.9896640827, "acceptance_length": 3.968992248, "accepted_per_position": [129, 128, 126] } } ] } ] }