File size: 16,427 Bytes
b78a22f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 | {
"schema_version": 15,
"artifact_type": "ninfer_bench_report",
"tool": "ninfer_bench",
"command": "/usr/local/bin/ninfer_bench --weights /models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer --corpus /workspace/ninfer/bench/fixtures/bench_corpus.ids --device 0 -n 256,1024 -pg 2048,512;8192,512 -r 3 --warmup 2 --max-ctx 16384 --prefill-chunk 1024 --kv-dtype fp8 --spec mtp --draft-tokens 5 --lm-head-draft --output json --output-file /out/mtp5-gpu1-r3.json",
"environment": {"gpu_name": "NVIDIA RTX PRO 4000 Blackwell", "cuda_runtime_version": "13.1", "cuda_driver_version": "13.2", "device_id": 0},
"artifact": {"path": "/models/thinkingcap-d48-o44-i53-g64-mtp-nativeq8-2026-09-27/thinkingcap-d48-o44-i53-g64-mtp-nativeq8.ninfer", "file_size_bytes": 17501292292},
"load": {
"architecture": "Qwen3_5ForCausalLM",
"name": "thinkingcap-qwen3.8-27b-d48-o44-i53-g64-mtp-nativeq8",
"formats": ["bf16","fp32","int32","nvfp4","q4_g64_fp16","q5_g64_fp16","q8_g32_fp16"],
"prefill_signature": "2047cd3b0a44f1986d5ae5cca67c16f458371663e9769b590d96582902ce65bc",
"load_seconds": 3.361672471,
"upload_seconds": 2.471311519,
"artifact_bytes_read": 17501379593,
"host_to_device_bytes": 17487933252,
"peak_staging_bytes": 268435456,
"device_object_count": 732,
"host_object_count": 4
},
"memory": {
"device": 0,
"max_context": 16384,
"kv_capacity_mode": "explicit",
"kv_capacity": 16384,
"kv_capacity_page_groups": 256,
"kv_capacity_max_page_groups": 256,
"kv_cache": "fp8-e4m3-row256",
"weights": {"capacity_bytes": 17487992064, "used_bytes": 17487992064, "peak_used_bytes": 17487992064},
"sequence": {"capacity_bytes": 755319040, "used_bytes": 755319040, "peak_used_bytes": 755319040},
"workspace": {"capacity_bytes": 159981568, "used_bytes": 0, "peak_used_bytes": 0},
"vision_workspace": null,
"minimum_runtime_reservation_bytes": 1001283840,
"kv_capacity_increment_bytes": 0,
"runtime_reservation_bytes": 1001283840,
"available_after_weights_bytes": 7420313600,
"available_after_startup_bytes": 6497566720,
"kv_capacity_headroom_bytes": 0,
"planned_slack_bytes": 6419029760,
"cuda_graph_allowance_bytes": 85983232,
"kv_payload_bytes": 575013888
},
"config": {
"max_context": 16384,
"prefill_chunk": 1024,
"kv_cache": "fp8-e4m3-row256",
"speculative_backend": "mtp",
"draft_tokens": 5,
"proposal_head": "optimized",
"use_cuda_graph": true,
"decode_path": "mtp_cuda_graph",
"decode_graph_prime": {"primed": true, "output_tokens": 13},
"repetitions": 3,
"warmup": 2,
"corpus_path": "/workspace/ninfer/bench/fixtures/bench_corpus.ids",
"corpus_tokens": 65536
},
"tests": [
{
"label": "tg256",
"kind": "tg",
"n_prompt": 0,
"n_gen": 256,
"requested_output_tokens": 257,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 45.64353647,
"decode_output_tok_s_stddev": 0.01225268331,
"decode_engine_tok_s_mean": 45.64353647,
"decode_engine_tok_s_stddev": 0.01225268331,
"prepare_seconds_mean": 1.626666667e-06,
"prepare_seconds_stddev": 4.163331999e-08,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 5.608680486,
"decode_seconds_stddev": 0.001505812499,
"total_seconds_mean": 5.646164623,
"total_seconds_stddev": 0.001530725286,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 366,
"drafted_tokens": 1818,
"accepted_tokens": 399,
"fallback_steps": 3,
"acceptance_rate": 0.2194719472,
"acceptance_length": 2.090163934,
"accepted_per_position": [195, 111, 60, 27, 6]
},
"reps": [
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.64e-06, "vision_seconds": 0, "prefill_seconds": 0.037218125, "decode_seconds": 5.608086056, "total_seconds": 5.645572101},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 122,
"drafted_tokens": 606,
"accepted_tokens": 133,
"fallback_steps": 1,
"acceptance_rate": 0.2194719472,
"acceptance_length": 2.090163934,
"accepted_per_position": [65, 37, 20, 9, 2]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.58e-06, "vision_seconds": 0, "prefill_seconds": 0.037194564, "decode_seconds": 5.607562617, "total_seconds": 5.64501873},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 122,
"drafted_tokens": 606,
"accepted_tokens": 133,
"fallback_steps": 1,
"acceptance_rate": 0.2194719472,
"acceptance_length": 2.090163934,
"accepted_per_position": [65, 37, 20, 9, 2]
}
},
{
"generated_output_tokens": 257,
"decode_output_tokens": 256,
"decode_engine_tokens": 256,
"timings": {"prepare_seconds": 1.66e-06, "vision_seconds": 0, "prefill_seconds": 0.037246264, "decode_seconds": 5.610392784, "total_seconds": 5.647903038},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 122,
"drafted_tokens": 606,
"accepted_tokens": 133,
"fallback_steps": 1,
"acceptance_rate": 0.2194719472,
"acceptance_length": 2.090163934,
"accepted_per_position": [65, 37, 20, 9, 2]
}
}
]
},
{
"label": "tg1024",
"kind": "tg",
"n_prompt": 0,
"n_gen": 1024,
"requested_output_tokens": 1025,
"prefill_tok_s_mean": null,
"prefill_tok_s_stddev": null,
"decode_output_tok_s_mean": 79.96223629,
"decode_output_tok_s_stddev": 0.01928508403,
"decode_engine_tok_s_mean": 79.96223629,
"decode_engine_tok_s_stddev": 0.01928508403,
"prepare_seconds_mean": 1.86e-06,
"prepare_seconds_stddev": 2.443358345e-07,
"prefill_seconds_mean": null,
"prefill_seconds_stddev": null,
"decode_seconds_mean": 12.80604554,
"decode_seconds_stddev": 0.003088215596,
"total_seconds_mean": 12.84390952,
"total_seconds_stddev": 0.003108343472,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 837,
"drafted_tokens": 4185,
"accepted_tokens": 2232,
"fallback_steps": 3,
"acceptance_rate": 0.5333333333,
"acceptance_length": 3.666666667,
"accepted_per_position": [636, 495, 405, 360, 336]
},
"reps": [
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 1.58e-06, "vision_seconds": 0, "prefill_seconds": 0.037275024, "decode_seconds": 12.80854159, "total_seconds": 12.84644666},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 279,
"drafted_tokens": 1395,
"accepted_tokens": 744,
"fallback_steps": 1,
"acceptance_rate": 0.5333333333,
"acceptance_length": 3.666666667,
"accepted_per_position": [212, 165, 135, 120, 112]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 1.97e-06, "vision_seconds": 0, "prefill_seconds": 0.037306394, "decode_seconds": 12.80700306, "total_seconds": 12.84483958},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 279,
"drafted_tokens": 1395,
"accepted_tokens": 744,
"fallback_steps": 1,
"acceptance_rate": 0.5333333333,
"acceptance_length": 3.666666667,
"accepted_per_position": [212, 165, 135, 120, 112]
}
},
{
"generated_output_tokens": 1025,
"decode_output_tokens": 1024,
"decode_engine_tokens": 1024,
"timings": {"prepare_seconds": 2.03e-06, "vision_seconds": 0, "prefill_seconds": 0.037319024, "decode_seconds": 12.80259198, "total_seconds": 12.84044232},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 279,
"drafted_tokens": 1395,
"accepted_tokens": 744,
"fallback_steps": 1,
"acceptance_rate": 0.5333333333,
"acceptance_length": 3.666666667,
"accepted_per_position": [212, 165, 135, 120, 112]
}
}
]
},
{
"label": "pp2048+tg512",
"kind": "pp+tg",
"n_prompt": 2048,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3578.647259,
"prefill_tok_s_stddev": 2.500665781,
"decode_output_tok_s_mean": 116.450161,
"decode_output_tok_s_stddev": 0.01153414308,
"decode_engine_tok_s_mean": 116.450161,
"decode_engine_tok_s_stddev": 0.01153414308,
"prepare_seconds_mean": 8.87e-06,
"prepare_seconds_stddev": 3.304542328e-07,
"prefill_seconds_mean": 0.5722834687,
"prefill_seconds_stddev": 0.0003998407674,
"decode_seconds_mean": 4.396730747,
"decode_seconds_stddev": 0.0004354861691,
"total_seconds_mean": 4.969295035,
"total_seconds_stddev": 0.0008332372725,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 285,
"drafted_tokens": 1416,
"accepted_tokens": 1251,
"fallback_steps": 0,
"acceptance_rate": 0.8834745763,
"acceptance_length": 5.389473684,
"accepted_per_position": [267, 258, 252, 243, 231]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 8.85e-06, "vision_seconds": 0, "prefill_seconds": 0.571859209, "decode_seconds": 4.396292774, "total_seconds": 4.968432922},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 95,
"drafted_tokens": 472,
"accepted_tokens": 417,
"fallback_steps": 0,
"acceptance_rate": 0.8834745763,
"acceptance_length": 5.389473684,
"accepted_per_position": [89, 86, 84, 81, 77]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 8.55e-06, "vision_seconds": 0, "prefill_seconds": 0.572337879, "decode_seconds": 4.396735763, "total_seconds": 4.969356152},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 95,
"drafted_tokens": 472,
"accepted_tokens": 417,
"fallback_steps": 0,
"acceptance_rate": 0.8834745763,
"acceptance_length": 5.389473684,
"accepted_per_position": [89, 86, 84, 81, 77]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 9.21e-06, "vision_seconds": 0, "prefill_seconds": 0.572653318, "decode_seconds": 4.397163703, "total_seconds": 4.970096031},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 95,
"drafted_tokens": 472,
"accepted_tokens": 417,
"fallback_steps": 0,
"acceptance_rate": 0.8834745763,
"acceptance_length": 5.389473684,
"accepted_per_position": [89, 86, 84, 81, 77]
}
}
]
},
{
"label": "pp8192+tg512",
"kind": "pp+tg",
"n_prompt": 8192,
"n_gen": 512,
"requested_output_tokens": 513,
"prefill_tok_s_mean": 3432.02866,
"prefill_tok_s_stddev": 1.682787014,
"decode_output_tok_s_mean": 125.793022,
"decode_output_tok_s_stddev": 0.07749953547,
"decode_engine_tok_s_mean": 125.793022,
"decode_engine_tok_s_stddev": 0.07749953547,
"prepare_seconds_mean": 2.550333333e-05,
"prepare_seconds_stddev": 2.935887146e-06,
"prefill_seconds_mean": 2.386926837,
"prefill_seconds_stddev": 0.001170184901,
"decode_seconds_mean": 4.070179105,
"decode_seconds_stddev": 0.002506856718,
"total_seconds_mean": 6.457397271,
"total_seconds_stddev": 0.002802127703,
"workspace_peak_bytes": 159981568,
"workspace_allocator_peak_bytes": 159981568,
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 261,
"drafted_tokens": 1302,
"accepted_tokens": 1275,
"fallback_steps": 0,
"acceptance_rate": 0.9792626728,
"acceptance_length": 5.885057471,
"accepted_per_position": [258, 258, 255, 255, 249]
},
"reps": [
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.738e-05, "vision_seconds": 0, "prefill_seconds": 2.387167149, "decode_seconds": 4.067344221, "total_seconds": 6.45479698},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 87,
"drafted_tokens": 434,
"accepted_tokens": 425,
"fallback_steps": 0,
"acceptance_rate": 0.9792626728,
"acceptance_length": 5.885057471,
"accepted_per_position": [86, 86, 85, 85, 83]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.701e-05, "vision_seconds": 0, "prefill_seconds": 2.385655151, "decode_seconds": 4.071089687, "total_seconds": 6.457029848},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 87,
"drafted_tokens": 434,
"accepted_tokens": 425,
"fallback_steps": 0,
"acceptance_rate": 0.9792626728,
"acceptance_length": 5.885057471,
"accepted_per_position": [86, 86, 85, 85, 83]
}
},
{
"generated_output_tokens": 513,
"decode_output_tokens": 512,
"decode_engine_tokens": 512,
"timings": {"prepare_seconds": 2.212e-05, "vision_seconds": 0, "prefill_seconds": 2.38795821, "decode_seconds": 4.072103406, "total_seconds": 6.460364985},
"speculative": {
"backend": "mtp",
"enabled": true,
"draft_window": 5,
"rounds": 87,
"drafted_tokens": 434,
"accepted_tokens": 425,
"fallback_steps": 0,
"acceptance_rate": 0.9792626728,
"acceptance_length": 5.885057471,
"accepted_per_position": [86, 86, 85, 85, 83]
}
}
]
}
]
}
|