{
  "schema_version": 15,
  "artifact_type": "ninfer_bench_report",
  "tool": "ninfer_bench",
  "command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --output json --output-file /out/mtp0-gpu1-r3.json",
  "environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0},
  "artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292},
  "load": {
    "architecture": "Qwen3_5ForCausalLM",
    "name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8",
    "formats": ["bf16","fp32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"],
    "prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc",
    "load_seconds": 3.215501918,
    "upload_seconds": 2.351215222,
    "artifact_bytes_read": 16692550665,
    "host_to_device_bytes": 16679625540,
    "peak_staging_bytes": 268435456,
    "device_object_count": 718,
    "host_object_count": 4
  },
  "memory": {
    "device": 0,
    "max_context": 16384,
    "kv_capacity_mode": "explicit",
    "kv_capacity": 16384,
    "kv_capacity_page_groups": 256,
    "kv_capacity_max_page_groups": 256,
    "kv_cache": "fp8-e4m3-row256",
    "weights": {"capacity_bytes": 16679684352, "used_bytes": 16679684352, "peak_used_bytes": 16679684352},
    "sequence": {"capacity_bytes": 707504896, "used_bytes": 707504896, "peak_used_bytes": 707504896},
    "workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0},
    "vision_workspace": null,
    "minimum_runtime_reservation_bytes": 880069376,
    "kv_capacity_increment_bytes": 0,
    "runtime_reservation_bytes": 880069376,
    "available_after_weights_bytes": 8227717120,
    "available_after_startup_bytes": 7355301888,
    "kv_capacity_headroom_bytes": 0,
    "planned_slack_bytes": 7347647744,
    "cuda_graph_allowance_bytes": 12582912,
    "kv_payload_bytes": 541065216
  },
  "config": {
    "max_context": 16384,
    "prefill_chunk": 1024,
    "kv_cache": "fp8-e4m3-row256",
    "speculative_backend": "none",
    "draft_tokens": 0,
    "proposal_head": "full",
    "use_cuda_graph": true,
    "decode_path": "cuda_graph",
    "decode_graph_prime": {"primed": true, "output_tokens": 3},
    "repetitions": 3,
    "warmup": 2,
    "corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids",
    "corpus_tokens": 65536
  },
  "tests": [
    {
      "label": "tg256",
      "kind": "tg",
      "n_prompt": 0,
      "n_gen": 256,
      "requested_output_tokens": 257,
      "prefill_tok_s_mean": null,
      "prefill_tok_s_stddev": null,
      "decode_output_tok_s_mean": 33.684761,
      "decode_output_tok_s_stddev": 0.1895257772,
      "decode_engine_tok_s_mean": 33.684761,
      "decode_engine_tok_s_stddev": 0.1895257772,
      "prepare_seconds_mean": 1.643333333e-06,
      "prepare_seconds_stddev": 1.594783162e-07,
      "prefill_seconds_mean": null,
      "prefill_seconds_stddev": null,
      "decode_seconds_mean": 7.600036103,
      "decode_seconds_stddev": 0.04273637934,
      "total_seconds_mean": 7.630126484,
      "total_seconds_stddev": 0.04289000507,
      "workspace_peak_bytes": 159981568,
      "workspace_allocator_peak_bytes": 159981568,
      "speculative": {
        "backend": "none",
        "enabled": false,
        "draft_window": 0,
        "rounds": 0,
        "drafted_tokens": 0,
        "accepted_tokens": 0,
        "fallback_steps": 0,
        "acceptance_rate": null,
        "acceptance_length": null,
        "accepted_per_position": []
      },
      "reps": [
        {
          "generated_output_tokens": 257,
          "decode_output_tokens": 256,
          "decode_engine_tokens": 256,
          "timings": {"prepare_seconds": 1.72e-06, "vision_seconds": 0, "prefill_seconds": 0.029530101, "decode_seconds": 7.555931071, "total_seconds": 7.585861902},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        },
        {
          "generated_output_tokens": 257,
          "decode_output_tokens": 256,
          "decode_engine_tokens": 256,
          "timings": {"prepare_seconds": 1.75e-06, "vision_seconds": 0, "prefill_seconds": 0.029666951, "decode_seconds": 7.602919433, "total_seconds": 7.633022412},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        },
        {
          "generated_output_tokens": 257,
          "decode_output_tokens": 256,
          "decode_engine_tokens": 256,
          "timings": {"prepare_seconds": 1.46e-06, "vision_seconds": 0, "prefill_seconds": 0.029836681, "decode_seconds": 7.641257806, "total_seconds": 7.671495137},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        }
      ]
    },
    {
      "label": "tg1024",
      "kind": "tg",
      "n_prompt": 0,
      "n_gen": 1024,
      "requested_output_tokens": 1025,
      "prefill_tok_s_mean": null,
      "prefill_tok_s_stddev": null,
      "decode_output_tok_s_mean": 33.56648354,
      "decode_output_tok_s_stddev": 0.07796054842,
      "decode_engine_tok_s_mean": 33.56648354,
      "decode_engine_tok_s_stddev": 0.07796054842,
      "prepare_seconds_mean": 2.236666667e-06,
      "prepare_seconds_stddev": 2.804163571e-07,
      "prefill_seconds_mean": null,
      "prefill_seconds_stddev": null,
      "decode_seconds_mean": 30.50673092,
      "decode_seconds_stddev": 0.07083484686,
      "total_seconds_mean": 30.53810507,
      "total_seconds_stddev": 0.07092740454,
      "workspace_peak_bytes": 159981568,
      "workspace_allocator_peak_bytes": 159981568,
      "speculative": {
        "backend": "none",
        "enabled": false,
        "draft_window": 0,
        "rounds": 0,
        "drafted_tokens": 0,
        "accepted_tokens": 0,
        "fallback_steps": 0,
        "acceptance_rate": null,
        "acceptance_length": null,
        "accepted_per_position": []
      },
      "reps": [
        {
          "generated_output_tokens": 1025,
          "decode_output_tokens": 1024,
          "decode_engine_tokens": 1024,
          "timings": {"prepare_seconds": 1.93e-06, "vision_seconds": 0, "prefill_seconds": 0.029781792, "decode_seconds": 30.43330908, "total_seconds": 30.46457702},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        },
        {
          "generated_output_tokens": 1025,
          "decode_output_tokens": 1024,
          "decode_engine_tokens": 1024,
          "timings": {"prepare_seconds": 2.48e-06, "vision_seconds": 0, "prefill_seconds": 0.029931181, "decode_seconds": 30.51222486, "total_seconds": 30.54362944},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        },
        {
          "generated_output_tokens": 1025,
          "decode_output_tokens": 1024,
          "decode_engine_tokens": 1024,
          "timings": {"prepare_seconds": 2.3e-06, "vision_seconds": 0, "prefill_seconds": 0.029967602, "decode_seconds": 30.57465883, "total_seconds": 30.60610875},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        }
      ]
    },
    {
      "label": "pp2048+tg512",
      "kind": "pp+tg",
      "n_prompt": 2048,
      "n_gen": 512,
      "requested_output_tokens": 513,
      "prefill_tok_s_mean": 3658.542902,
      "prefill_tok_s_stddev": 3.445873284,
      "decode_output_tok_s_mean": 33.28000528,
      "decode_output_tok_s_stddev": 0.006441816527,
      "decode_engine_tok_s_mean": 33.28000528,
      "decode_engine_tok_s_stddev": 0.006441816527,
      "prepare_seconds_mean": 1.159666667e-05,
      "prepare_seconds_stddev": 5.299371032e-07,
      "prefill_seconds_mean": 0.5597860313,
      "prefill_seconds_stddev": 0.0005275218527,
      "decode_seconds_mean": 15.38461333,
      "decode_seconds_stddev": 0.002977803412,
      "total_seconds_mean": 15.9452091,
      "total_seconds_stddev": 0.003136026081,
      "workspace_peak_bytes": 159981568,
      "workspace_allocator_peak_bytes": 159981568,
      "speculative": {
        "backend": "none",
        "enabled": false,
        "draft_window": 0,
        "rounds": 0,
        "drafted_tokens": 0,
        "accepted_tokens": 0,
        "fallback_steps": 0,
        "acceptance_rate": null,
        "acceptance_length": null,
        "accepted_per_position": []
      },
      "reps": [
        {
          "generated_output_tokens": 513,
          "decode_output_tokens": 512,
          "decode_engine_tokens": 512,
          "timings": {"prepare_seconds": 1.103e-05, "vision_seconds": 0, "prefill_seconds": 0.559531288, "decode_seconds": 15.38738746, "total_seconds": 15.94772397},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        },
        {
          "generated_output_tokens": 513,
          "decode_output_tokens": 512,
          "decode_engine_tokens": 512,
          "timings": {"prepare_seconds": 1.168e-05, "vision_seconds": 0, "prefill_seconds": 0.560392578, "decode_seconds": 15.38498566, "total_seconds": 15.94620803},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        },
        {
          "generated_output_tokens": 513,
          "decode_output_tokens": 512,
          "decode_engine_tokens": 512,
          "timings": {"prepare_seconds": 1.208e-05, "vision_seconds": 0, "prefill_seconds": 0.559434228, "decode_seconds": 15.38146687, "total_seconds": 15.94169529},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        }
      ]
    },
    {
      "label": "pp8192+tg512",
      "kind": "pp+tg",
      "n_prompt": 8192,
      "n_gen": 512,
      "requested_output_tokens": 513,
      "prefill_tok_s_mean": 3474.769135,
      "prefill_tok_s_stddev": 1.539211318,
      "decode_output_tok_s_mean": 32.77703692,
      "decode_output_tok_s_stddev": 0.009201321588,
      "decode_engine_tok_s_mean": 32.77703692,
      "decode_engine_tok_s_stddev": 0.009201321588,
      "prepare_seconds_mean": 2.889e-05,
      "prepare_seconds_stddev": 2.343074903e-07,
      "prefill_seconds_mean": 2.357567008,
      "prefill_seconds_stddev": 0.001044130902,
      "decode_seconds_mean": 15.62069287,
      "decode_seconds_stddev": 0.004384481473,
      "total_seconds_mean": 17.97908915,
      "total_seconds_stddev": 0.005416429425,
      "workspace_peak_bytes": 159981568,
      "workspace_allocator_peak_bytes": 159981568,
      "speculative": {
        "backend": "none",
        "enabled": false,
        "draft_window": 0,
        "rounds": 0,
        "drafted_tokens": 0,
        "accepted_tokens": 0,
        "fallback_steps": 0,
        "acceptance_rate": null,
        "acceptance_length": null,
        "accepted_per_position": []
      },
      "reps": [
        {
          "generated_output_tokens": 513,
          "decode_output_tokens": 512,
          "decode_engine_tokens": 512,
          "timings": {"prepare_seconds": 2.877e-05, "vision_seconds": 0, "prefill_seconds": 2.356398688, "decode_seconds": 15.61569361, "total_seconds": 17.9729261},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        },
        {
          "generated_output_tokens": 513,
          "decode_output_tokens": 512,
          "decode_engine_tokens": 512,
          "timings": {"prepare_seconds": 2.874e-05, "vision_seconds": 0, "prefill_seconds": 2.357893328, "decode_seconds": 15.6225002, "total_seconds": 17.98124853},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        },
        {
          "generated_output_tokens": 513,
          "decode_output_tokens": 512,
          "decode_engine_tokens": 512,
          "timings": {"prepare_seconds": 2.916e-05, "vision_seconds": 0, "prefill_seconds": 2.358409007, "decode_seconds": 15.62388479, "total_seconds": 17.98309282},
          "speculative": {
            "backend": "none",
            "enabled": false,
            "draft_window": 0,
            "rounds": 0,
            "drafted_tokens": 0,
            "accepted_tokens": 0,
            "fallback_steps": 0,
            "acceptance_rate": null,
            "acceptance_length": null,
            "accepted_per_position": []
          }
        }
      ]
    }
  ]
}