File size: 16,315 Bytes
b78a22f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 | {
"schema_version": 15,
"artifact_type": "ninfer_bench_report",
"tool": "ninfer_bench",
"command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 3 --lm-head-draft --output json --output-file /out/mtp3-gpu1-r3.json",
"environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0},
"artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292},
"load": {
"architecture": "Qwen3_5ForCausalLM",
"name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8",
"formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"],
"prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc",
"load_seconds": 3.316723434,
"upload_seconds": 2.466533213,
"artifact_bytes_read": 17501379593,
"host_to_device_bytes": 17487933252,
"peak_staging_bytes": 268435456,
"device_object_count": 732,
"host_object_count": 4
},
"memory": {
"device": 0,
"max_context": 16384,
"kv_capacity_mode": "explicit",
"kv_capacity": 16384,
"kv_capacity_page_groups": 256,
"kv_capacity_max_page_groups": 256,
"kv_cache": "fp8-e4m3-row256",
"weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064},
"sequence": {"capacity_bytes": 750708992, "used_bytes": 750708992, "peak_used_bytes": 750708992},
"workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0},
"vision_workspace": null,
"minimum_runtime_reservation_bytes": 996673792,
"kv_capacity_increment_bytes": 0,
"runtime_reservation_bytes": 996673792,
"available_after_weights_bytes": 7420313600,
"available_after_startup_bytes": 6505955328,
"kv_capacity_headroom_bytes": 0,
"planned_slack_bytes": 6423639808,
"cuda_graph_allowance_bytes": 85983232,
"kv_payload_bytes": 575013888
},
"config": {
"max_context": 16384,
"prefill_chunk": 1024,
"kv_cache": "fp8-e4m3-row256",
"speculative_backend": "mtp",
"draft_tokens": 3,
"proposal_head": "optimized",
"use_cuda_graph": true,
"decode_path": "mtp_cuda_graph",
"decode_graph_prime": {"primed": true, "output_tokens": 9},
"repetitions": 3,
"warmup": 2,
"corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids",
"corpus_tokens": 65536
},
"tests": [
{
"label": "tg256",
"kind": "tg",
"n_prompt": 0,
"n_gen": 256,
"requested_output_tokens": 257,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 54.16346999,
"decode_output_tok_s_stddev": 0.02061488267,
"decode_engine_tok_s_mean": 54.16346999,
"decode_engine_tok_s_stddev": 0.02061488267,
"prepare_seconds_mean": 1.31e-06,
"prepare_seconds_stddev": 2.351595203e-07,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 4.726433236,
"decode_seconds_stddev": 0.00179857864,
"total_seconds_mean": 4.760920559,
"total_seconds_stddev": 0.001817732811,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 342,
"drafted_tokens": 1023,
"accepted_tokens": 426,
"fallback_steps": 0,
"acceptance_rate": 0.4164222874,
"acceptance_length": 2.245614035,
"accepted_per_position": [219, 141, 66]
},
"reps": [
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.3e-06, "vision_seconds": 0, "prefill_seconds": 0.034216187, "decode_seconds": 4.724398538, "total_seconds": 4.758862795},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 114,
"drafted_tokens": 341,
"accepted_tokens": 142,
"fallback_steps": 0,
"acceptance_rate": 0.4164222874,
"acceptance_length": 2.245614035,
"accepted_per_position": [73, 47, 22]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.08e-06, "vision_seconds": 0, "prefill_seconds": 0.034256036, "decode_seconds": 4.727810985, "total_seconds": 4.762307711},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 114,
"drafted_tokens": 341,
"accepted_tokens": 142,
"fallback_steps": 0,
"acceptance_rate": 0.4164222874,
"acceptance_length": 2.245614035,
"accepted_per_position": [73, 47, 22]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.55e-06, "vision_seconds": 0, "prefill_seconds": 0.034265667, "decode_seconds": 4.727090185, "total_seconds": 4.761591172},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 114,
"drafted_tokens": 341,
"accepted_tokens": 142,
"fallback_steps": 0,
"acceptance_rate": 0.4164222874,
"acceptance_length": 2.245614035,
"accepted_per_position": [73, 47, 22]
}
}
]
},
{
"label": "tg1024",
"kind": "tg",
"n_prompt": 0,
"n_gen": 1024,
"requested_output_tokens": 1025,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 58.0845176,
"decode_output_tok_s_stddev": 0.00498641247,
"decode_engine_tok_s_mean": 58.0845176,
"decode_engine_tok_s_stddev": 0.00498641247,
"prepare_seconds_mean": 1.92e-06,
"prepare_seconds_stddev": 6.850547423e-07,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 17.62948282,
"decode_seconds_stddev": 0.001513520352,
"total_seconds_mean": 17.66459143,
"total_seconds_stddev": 0.001473746939,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 1269,
"drafted_tokens": 3804,
"accepted_tokens": 1800,
"fallback_steps": 3,
"acceptance_rate": 0.4731861199,
"acceptance_length": 2.418439716,
"accepted_per_position": [909, 576, 315]
},
"reps": [
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 1.13e-06, "vision_seconds": 0, "prefill_seconds": 0.034378477, "decode_seconds": 17.62873566, "total_seconds": 17.66394648},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 423,
"drafted_tokens": 1268,
"accepted_tokens": 600,
"fallback_steps": 1,
"acceptance_rate": 0.4731861199,
"acceptance_length": 2.418439716,
"accepted_per_position": [303, 192, 105]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 2.28e-06, "vision_seconds": 0, "prefill_seconds": 0.034313017, "decode_seconds": 17.63122463, "total_seconds": 17.66627771},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 423,
"drafted_tokens": 1268,
"accepted_tokens": 600,
"fallback_steps": 1,
"acceptance_rate": 0.4731861199,
"acceptance_length": 2.418439716,
"accepted_per_position": [303, 192, 105]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 2.35e-06, "vision_seconds": 0, "prefill_seconds": 0.034337167, "decode_seconds": 17.62848816, "total_seconds": 17.6635501},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 423,
"drafted_tokens": 1268,
"accepted_tokens": 600,
"fallback_steps": 1,
"acceptance_rate": 0.4731861199,
"acceptance_length": 2.418439716,
"accepted_per_position": [303, 192, 105]
}
}
]
},
{
"label": "pp2048+tg512",
"kind": "pp+tg",
"n_prompt": 2048,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3599.547483,
"prefill_tok_s_stddev": 2.252708112,
"decode_output_tok_s_mean": 89.6428123,
"decode_output_tok_s_stddev": 0.02176244083,
"decode_engine_tok_s_mean": 89.6428123,
"decode_engine_tok_s_stddev": 0.02176244083,
"prepare_seconds_mean": 9.456666667e-06,
"prepare_seconds_stddev": 1.209021643e-06,
"prefill_seconds_mean": 0.5689605553,
"prefill_seconds_stddev": 0.0003559983107,
"decode_seconds_mean": 5.711556866,
"decode_seconds_stddev": 0.00138663725,
"total_seconds_mean": 6.280836125,
"total_seconds_stddev": 0.001156129626,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 408,
"drafted_tokens": 1221,
"accepted_tokens": 1128,
"fallback_steps": 0,
"acceptance_rate": 0.9238329238,
"acceptance_length": 3.764705882,
"accepted_per_position": [384, 378, 366]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 1.029e-05, "vision_seconds": 0, "prefill_seconds": 0.568569952, "decode_seconds": 5.713009855, "total_seconds": 6.281900687},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 136,
"drafted_tokens": 407,
"accepted_tokens": 376,
"fallback_steps": 0,
"acceptance_rate": 0.9238329238,
"acceptance_length": 3.764705882,
"accepted_per_position": [128, 126, 122]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 8.07e-06, "vision_seconds": 0, "prefill_seconds": 0.569044922, "decode_seconds": 5.710247808, "total_seconds": 6.279606229},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 136,
"drafted_tokens": 407,
"accepted_tokens": 376,
"fallback_steps": 0,
"acceptance_rate": 0.9238329238,
"acceptance_length": 3.764705882,
"accepted_per_position": [128, 126, 122]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 1.001e-05, "vision_seconds": 0, "prefill_seconds": 0.569266792, "decode_seconds": 5.711412936, "total_seconds": 6.281001458},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 136,
"drafted_tokens": 407,
"accepted_tokens": 376,
"fallback_steps": 0,
"acceptance_rate": 0.9238329238,
"acceptance_length": 3.764705882,
"accepted_per_position": [128, 126, 122]
}
}
]
},
{
"label": "pp8192+tg512",
"kind": "pp+tg",
"n_prompt": 8192,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3436.856537,
"prefill_tok_s_stddev": 0.2878826734,
"decode_output_tok_s_mean": 93.5343478,
"decode_output_tok_s_stddev": 0.02561220332,
"decode_engine_tok_s_mean": 93.5343478,
"decode_engine_tok_s_stddev": 0.02561220332,
"prepare_seconds_mean": 2.846666667e-05,
"prepare_seconds_stddev": 1.435142269e-06,
"prefill_seconds_mean": 2.383573463,
"prefill_seconds_stddev": 0.0001996630351,
"decode_seconds_mean": 5.473925222,
"decode_seconds_stddev": 0.001498824652,
"total_seconds_mean": 7.857857098,
"total_seconds_stddev": 0.001348104877,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 387,
"drafted_tokens": 1161,
"accepted_tokens": 1149,
"fallback_steps": 0,
"acceptance_rate": 0.9896640827,
"acceptance_length": 3.968992248,
"accepted_per_position": [387, 384, 378]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.783e-05, "vision_seconds": 0, "prefill_seconds": 2.383513023, "decode_seconds": 5.475311554, "total_seconds": 7.859187347},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 129,
"drafted_tokens": 387,
"accepted_tokens": 383,
"fallback_steps": 0,
"acceptance_rate": 0.9896640827,
"acceptance_length": 3.968992248,
"accepted_per_position": [129, 128, 126]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 3.011e-05, "vision_seconds": 0, "prefill_seconds": 2.383411003, "decode_seconds": 5.474129296, "total_seconds": 7.857892128},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 129,
"drafted_tokens": 387,
"accepted_tokens": 383,
"fallback_steps": 0,
"acceptance_rate": 0.9896640827,
"acceptance_length": 3.968992248,
"accepted_per_position": [129, 128, 126]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.746e-05, "vision_seconds": 0, "prefill_seconds": 2.383796363, "decode_seconds": 5.472334817, "total_seconds": 7.85649182},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 3,
"rounds": 129,
"drafted_tokens": 387,
"accepted_tokens": 383,
"fallback_steps": 0,
"acceptance_rate": 0.9896640827,
"acceptance_length": 3.968992248,
"accepted_per_position": [129, 128, 126]
}
}
]
}
]
}
|