| { |
| "schema_version": 15, |
| "artifact_type": "ninfer_bench_report", |
| "tool": "ninfer_bench", |
| "command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --output json --output-file /out/mtp0-gpu1-r3.json", |
| "environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0}, |
| "artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292}, |
| "load": { |
| "architecture": "Qwen3_5ForCausalLM", |
| "name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8", |
| "formats": ["bf16","fp32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"], |
| "prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc", |
| "load_seconds": 3.215501918, |
| "upload_seconds": 2.351215222, |
| "artifact_bytes_read": 16692550665, |
| "host_to_device_bytes": 16679625540, |
| "peak_staging_bytes": 268435456, |
| "device_object_count": 718, |
| "host_object_count": 4 |
| }, |
| "memory": { |
| "device": 0, |
| "max_context": 16384, |
| "kv_capacity_mode": "explicit", |
| "kv_capacity": 16384, |
| "kv_capacity_page_groups": 256, |
| "kv_capacity_max_page_groups": 256, |
| "kv_cache": "fp8-e4m3-row256", |
| "weights": {"capacity_bytes": 16679684352, "used_bytes": 16679684352, "peak_used_bytes": 16679684352}, |
| "sequence": {"capacity_bytes": 707504896, "used_bytes": 707504896, "peak_used_bytes": 707504896}, |
| "workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0}, |
| "vision_workspace": null, |
| "minimum_runtime_reservation_bytes": 880069376, |
| "kv_capacity_increment_bytes": 0, |
| "runtime_reservation_bytes": 880069376, |
| "available_after_weights_bytes": 8227717120, |
| "available_after_startup_bytes": 7355301888, |
| "kv_capacity_headroom_bytes": 0, |
| "planned_slack_bytes": 7347647744, |
| "cuda_graph_allowance_bytes": 12582912, |
| "kv_payload_bytes": 541065216 |
| }, |
| "config": { |
| "max_context": 16384, |
| "prefill_chunk": 1024, |
| "kv_cache": "fp8-e4m3-row256", |
| "speculative_backend": "none", |
| "draft_tokens": 0, |
| "proposal_head": "full", |
| "use_cuda_graph": true, |
| "decode_path": "cuda_graph", |
| "decode_graph_prime": {"primed": true, "output_tokens": 3}, |
| "repetitions": 3, |
| "warmup": 2, |
| "corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids", |
| "corpus_tokens": 65536 |
| }, |
| "tests": [ |
| { |
| "label": "tg256", |
| "kind": "tg", |
| "n_prompt": 0, |
| "n_gen": 256, |
| "requested_output_tokens": 257, |
| "prefill_tok_s_mean": null, |
| "prefill_tok_s_stddev": null, |
| "decode_output_tok_s_mean": 33.684761, |
| "decode_output_tok_s_stddev": 0.1895257772, |
| "decode_engine_tok_s_mean": 33.684761, |
| "decode_engine_tok_s_stddev": 0.1895257772, |
| "prepare_seconds_mean": 1.643333333e-06, |
| "prepare_seconds_stddev": 1.594783162e-07, |
| "prefill_seconds_mean": null, |
| "prefill_seconds_stddev": null, |
| "decode_seconds_mean": 7.600036103, |
| "decode_seconds_stddev": 0.04273637934, |
| "total_seconds_mean": 7.630126484, |
| "total_seconds_stddev": 0.04289000507, |
| "workspace_peak_bytes": 159981568, |
| "workspace_allocator_peak_bytes": 159981568, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| }, |
| "reps": [ |
| { |
| "generated_output_tokens": 257, |
| "decode_output_tokens": 256, |
| "decode_engine_tokens": 256, |
| "timings": {"prepare_seconds": 1.72e-06, "vision_seconds": 0, "prefill_seconds": 0.029530101, "decode_seconds": 7.555931071, "total_seconds": 7.585861902}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| }, |
| { |
| "generated_output_tokens": 257, |
| "decode_output_tokens": 256, |
| "decode_engine_tokens": 256, |
| "timings": {"prepare_seconds": 1.75e-06, "vision_seconds": 0, "prefill_seconds": 0.029666951, "decode_seconds": 7.602919433, "total_seconds": 7.633022412}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| }, |
| { |
| "generated_output_tokens": 257, |
| "decode_output_tokens": 256, |
| "decode_engine_tokens": 256, |
| "timings": {"prepare_seconds": 1.46e-06, "vision_seconds": 0, "prefill_seconds": 0.029836681, "decode_seconds": 7.641257806, "total_seconds": 7.671495137}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| } |
| ] |
| }, |
| { |
| "label": "tg1024", |
| "kind": "tg", |
| "n_prompt": 0, |
| "n_gen": 1024, |
| "requested_output_tokens": 1025, |
| "prefill_tok_s_mean": null, |
| "prefill_tok_s_stddev": null, |
| "decode_output_tok_s_mean": 33.56648354, |
| "decode_output_tok_s_stddev": 0.07796054842, |
| "decode_engine_tok_s_mean": 33.56648354, |
| "decode_engine_tok_s_stddev": 0.07796054842, |
| "prepare_seconds_mean": 2.236666667e-06, |
| "prepare_seconds_stddev": 2.804163571e-07, |
| "prefill_seconds_mean": null, |
| "prefill_seconds_stddev": null, |
| "decode_seconds_mean": 30.50673092, |
| "decode_seconds_stddev": 0.07083484686, |
| "total_seconds_mean": 30.53810507, |
| "total_seconds_stddev": 0.07092740454, |
| "workspace_peak_bytes": 159981568, |
| "workspace_allocator_peak_bytes": 159981568, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| }, |
| "reps": [ |
| { |
| "generated_output_tokens": 1025, |
| "decode_output_tokens": 1024, |
| "decode_engine_tokens": 1024, |
| "timings": {"prepare_seconds": 1.93e-06, "vision_seconds": 0, "prefill_seconds": 0.029781792, "decode_seconds": 30.43330908, "total_seconds": 30.46457702}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| }, |
| { |
| "generated_output_tokens": 1025, |
| "decode_output_tokens": 1024, |
| "decode_engine_tokens": 1024, |
| "timings": {"prepare_seconds": 2.48e-06, "vision_seconds": 0, "prefill_seconds": 0.029931181, "decode_seconds": 30.51222486, "total_seconds": 30.54362944}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| }, |
| { |
| "generated_output_tokens": 1025, |
| "decode_output_tokens": 1024, |
| "decode_engine_tokens": 1024, |
| "timings": {"prepare_seconds": 2.3e-06, "vision_seconds": 0, "prefill_seconds": 0.029967602, "decode_seconds": 30.57465883, "total_seconds": 30.60610875}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| } |
| ] |
| }, |
| { |
| "label": "pp2048+tg512", |
| "kind": "pp+tg", |
| "n_prompt": 2048, |
| "n_gen": 512, |
| "requested_output_tokens": 513, |
| "prefill_tok_s_mean": 3658.542902, |
| "prefill_tok_s_stddev": 3.445873284, |
| "decode_output_tok_s_mean": 33.28000528, |
| "decode_output_tok_s_stddev": 0.006441816527, |
| "decode_engine_tok_s_mean": 33.28000528, |
| "decode_engine_tok_s_stddev": 0.006441816527, |
| "prepare_seconds_mean": 1.159666667e-05, |
| "prepare_seconds_stddev": 5.299371032e-07, |
| "prefill_seconds_mean": 0.5597860313, |
| "prefill_seconds_stddev": 0.0005275218527, |
| "decode_seconds_mean": 15.38461333, |
| "decode_seconds_stddev": 0.002977803412, |
| "total_seconds_mean": 15.9452091, |
| "total_seconds_stddev": 0.003136026081, |
| "workspace_peak_bytes": 159981568, |
| "workspace_allocator_peak_bytes": 159981568, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| }, |
| "reps": [ |
| { |
| "generated_output_tokens": 513, |
| "decode_output_tokens": 512, |
| "decode_engine_tokens": 512, |
| "timings": {"prepare_seconds": 1.103e-05, "vision_seconds": 0, "prefill_seconds": 0.559531288, "decode_seconds": 15.38738746, "total_seconds": 15.94772397}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| }, |
| { |
| "generated_output_tokens": 513, |
| "decode_output_tokens": 512, |
| "decode_engine_tokens": 512, |
| "timings": {"prepare_seconds": 1.168e-05, "vision_seconds": 0, "prefill_seconds": 0.560392578, "decode_seconds": 15.38498566, "total_seconds": 15.94620803}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| }, |
| { |
| "generated_output_tokens": 513, |
| "decode_output_tokens": 512, |
| "decode_engine_tokens": 512, |
| "timings": {"prepare_seconds": 1.208e-05, "vision_seconds": 0, "prefill_seconds": 0.559434228, "decode_seconds": 15.38146687, "total_seconds": 15.94169529}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| } |
| ] |
| }, |
| { |
| "label": "pp8192+tg512", |
| "kind": "pp+tg", |
| "n_prompt": 8192, |
| "n_gen": 512, |
| "requested_output_tokens": 513, |
| "prefill_tok_s_mean": 3474.769135, |
| "prefill_tok_s_stddev": 1.539211318, |
| "decode_output_tok_s_mean": 32.77703692, |
| "decode_output_tok_s_stddev": 0.009201321588, |
| "decode_engine_tok_s_mean": 32.77703692, |
| "decode_engine_tok_s_stddev": 0.009201321588, |
| "prepare_seconds_mean": 2.889e-05, |
| "prepare_seconds_stddev": 2.343074903e-07, |
| "prefill_seconds_mean": 2.357567008, |
| "prefill_seconds_stddev": 0.001044130902, |
| "decode_seconds_mean": 15.62069287, |
| "decode_seconds_stddev": 0.004384481473, |
| "total_seconds_mean": 17.97908915, |
| "total_seconds_stddev": 0.005416429425, |
| "workspace_peak_bytes": 159981568, |
| "workspace_allocator_peak_bytes": 159981568, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| }, |
| "reps": [ |
| { |
| "generated_output_tokens": 513, |
| "decode_output_tokens": 512, |
| "decode_engine_tokens": 512, |
| "timings": {"prepare_seconds": 2.877e-05, "vision_seconds": 0, "prefill_seconds": 2.356398688, "decode_seconds": 15.61569361, "total_seconds": 17.9729261}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| }, |
| { |
| "generated_output_tokens": 513, |
| "decode_output_tokens": 512, |
| "decode_engine_tokens": 512, |
| "timings": {"prepare_seconds": 2.874e-05, "vision_seconds": 0, "prefill_seconds": 2.357893328, "decode_seconds": 15.6225002, "total_seconds": 17.98124853}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| }, |
| { |
| "generated_output_tokens": 513, |
| "decode_output_tokens": 512, |
| "decode_engine_tokens": 512, |
| "timings": {"prepare_seconds": 2.916e-05, "vision_seconds": 0, "prefill_seconds": 2.358409007, "decode_seconds": 15.62388479, "total_seconds": 17.98309282}, |
| "speculative": { |
| "backend": "none", |
| "enabled": false, |
| "draft_window": 0, |
| "rounds": 0, |
| "drafted_tokens": 0, |
| "accepted_tokens": 0, |
| "fallback_steps": 0, |
| "acceptance_rate": null, |
| "acceptance_length": null, |
| "accepted_per_position": [] |
| } |
| } |
| ] |
| } |
| ] |
| } |
|
|
