| { |
| "runtime": "vLLM main 7ca49fbe4bab019e55d57cdc4b7fd3d55c67c1a6", |
| "gpu": "RTX 3090 24 GB (physical CUDA:1 in the reference run)", |
| "profile": { |
| "max_model_len": 1024, |
| "max_num_seqs": 32, |
| "max_num_batched_tokens": 8192, |
| "gpu_memory_utilization": 0.96, |
| "cuda_graphs": true, |
| "prefix_caching": false, |
| "limit_mm_per_prompt": { |
| "image": 1, |
| "video": 0 |
| } |
| }, |
| "method": "Three repetitions per point; short unique prompts; fixed 128-token greedy ignore-EOS outputs; output tokens divided by end-to-end batch completion time after warmup.", |
| "records": [ |
| { |
| "workload": "text", |
| "concurrency": 1, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 384, |
| "aggregate_output_tok_s": 156.57567204605067, |
| "per_request_output_tok_s": 156.57567204605067, |
| "mean_completion_seconds": 0.8174960920005105, |
| "mean_tpot_ms": 6.386688218753989, |
| "peak_observed_gpu_memory_mib": 23010, |
| "runs": [ |
| { |
| "generated_tokens": 128, |
| "wall_seconds": 0.81746169104008, |
| "output_tok_s": 156.58226116644306, |
| "gpu_memory_mib_after": 23010 |
| }, |
| { |
| "generated_tokens": 128, |
| "wall_seconds": 0.8176816419581883, |
| "output_tok_s": 156.54014158060946, |
| "gpu_memory_mib_after": 23010 |
| }, |
| { |
| "generated_tokens": 128, |
| "wall_seconds": 0.8173449430032633, |
| "output_tok_s": 156.60462708642333, |
| "gpu_memory_mib_after": 23010 |
| } |
| ] |
| }, |
| { |
| "workload": "one_image_per_request", |
| "concurrency": 1, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 384, |
| "aggregate_output_tok_s": 127.90552416322762, |
| "per_request_output_tok_s": 127.90552416322762, |
| "mean_completion_seconds": 1.0007386376576808, |
| "mean_tpot_ms": 7.818270606700632, |
| "peak_observed_gpu_memory_mib": 23010, |
| "runs": [ |
| { |
| "generated_tokens": 128, |
| "wall_seconds": 0.9040946460445412, |
| "output_tok_s": 141.5780975587084, |
| "gpu_memory_mib_after": 23010 |
| }, |
| { |
| "generated_tokens": 128, |
| "wall_seconds": 1.0513607729808427, |
| "output_tok_s": 121.74698095030823, |
| "gpu_memory_mib_after": 23010 |
| }, |
| { |
| "generated_tokens": 128, |
| "wall_seconds": 1.0467604939476587, |
| "output_tok_s": 122.28203179246121, |
| "gpu_memory_mib_after": 23010 |
| } |
| ] |
| }, |
| { |
| "workload": "text", |
| "concurrency": 4, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 1536, |
| "aggregate_output_tok_s": 486.91511429228393, |
| "per_request_output_tok_s": 121.72877857307098, |
| "mean_completion_seconds": 1.0515179853149068, |
| "mean_tpot_ms": 8.21498426027271, |
| "peak_observed_gpu_memory_mib": 23054, |
| "runs": [ |
| { |
| "generated_tokens": 512, |
| "wall_seconds": 1.0211215899907984, |
| "output_tok_s": 501.409435486144, |
| "gpu_memory_mib_after": 23010 |
| }, |
| { |
| "generated_tokens": 512, |
| "wall_seconds": 1.0497098179766908, |
| "output_tok_s": 487.7538451406283, |
| "gpu_memory_mib_after": 23010 |
| }, |
| { |
| "generated_tokens": 512, |
| "wall_seconds": 1.0837225479772314, |
| "output_tok_s": 472.4456466792614, |
| "gpu_memory_mib_after": 23054 |
| } |
| ] |
| }, |
| { |
| "workload": "one_image_per_request", |
| "concurrency": 4, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 1536, |
| "aggregate_output_tok_s": 262.6200812679751, |
| "per_request_output_tok_s": 65.65502031699377, |
| "mean_completion_seconds": 1.949584348340674, |
| "mean_tpot_ms": 15.231127721411514, |
| "peak_observed_gpu_memory_mib": 23054, |
| "runs": [ |
| { |
| "generated_tokens": 512, |
| "wall_seconds": 1.9419829859980382, |
| "output_tok_s": 263.64803589505664, |
| "gpu_memory_mib_after": 23010 |
| }, |
| { |
| "generated_tokens": 512, |
| "wall_seconds": 1.9698979230015539, |
| "output_tok_s": 259.911944685875, |
| "gpu_memory_mib_after": 23054 |
| }, |
| { |
| "generated_tokens": 512, |
| "wall_seconds": 1.93687213602243, |
| "output_tok_s": 264.3437274344014, |
| "gpu_memory_mib_after": 23054 |
| } |
| ] |
| }, |
| { |
| "workload": "text", |
| "concurrency": 8, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 3072, |
| "aggregate_output_tok_s": 820.4449600227697, |
| "per_request_output_tok_s": 102.55562000284621, |
| "mean_completion_seconds": 1.2481032243425336, |
| "mean_tpot_ms": 9.750806440176044, |
| "peak_observed_gpu_memory_mib": 23138, |
| "runs": [ |
| { |
| "generated_tokens": 1024, |
| "wall_seconds": 1.2773179819923826, |
| "output_tok_s": 801.6797809444029, |
| "gpu_memory_mib_after": 23054 |
| }, |
| { |
| "generated_tokens": 1024, |
| "wall_seconds": 1.277832499006763, |
| "output_tok_s": 801.3569859867684, |
| "gpu_memory_mib_after": 23138 |
| }, |
| { |
| "generated_tokens": 1024, |
| "wall_seconds": 1.1891591920284554, |
| "output_tok_s": 861.1126305581269, |
| "gpu_memory_mib_after": 23138 |
| } |
| ] |
| }, |
| { |
| "workload": "one_image_per_request", |
| "concurrency": 8, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 3072, |
| "aggregate_output_tok_s": 429.9295467387298, |
| "per_request_output_tok_s": 53.74119334234123, |
| "mean_completion_seconds": 2.3817855920060538, |
| "mean_tpot_ms": 18.607699937547295, |
| "peak_observed_gpu_memory_mib": 23138, |
| "runs": [ |
| { |
| "generated_tokens": 1024, |
| "wall_seconds": 1.7241748640080914, |
| "output_tok_s": 593.9072778381453, |
| "gpu_memory_mib_after": 23138 |
| }, |
| { |
| "generated_tokens": 1024, |
| "wall_seconds": 2.370788298023399, |
| "output_tok_s": 431.92384611217335, |
| "gpu_memory_mib_after": 23138 |
| }, |
| { |
| "generated_tokens": 1024, |
| "wall_seconds": 3.050393613986671, |
| "output_tok_s": 335.6943822937319, |
| "gpu_memory_mib_after": 23138 |
| } |
| ] |
| }, |
| { |
| "workload": "text", |
| "concurrency": 16, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 6144, |
| "aggregate_output_tok_s": 1269.7205057297842, |
| "per_request_output_tok_s": 79.35753160811151, |
| "mean_completion_seconds": 1.6129533946708154, |
| "mean_tpot_ms": 12.601198395865746, |
| "peak_observed_gpu_memory_mib": 23430, |
| "runs": [ |
| { |
| "generated_tokens": 2048, |
| "wall_seconds": 1.631988351000473, |
| "output_tok_s": 1254.9109181719928, |
| "gpu_memory_mib_after": 23138 |
| }, |
| { |
| "generated_tokens": 2048, |
| "wall_seconds": 1.5828196799848229, |
| "output_tok_s": 1293.8934395986519, |
| "gpu_memory_mib_after": 23430 |
| }, |
| { |
| "generated_tokens": 2048, |
| "wall_seconds": 1.6240521530271508, |
| "output_tok_s": 1261.0432467840592, |
| "gpu_memory_mib_after": 23430 |
| } |
| ] |
| }, |
| { |
| "workload": "one_image_per_request", |
| "concurrency": 16, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 6144, |
| "aggregate_output_tok_s": 518.5347963568223, |
| "per_request_output_tok_s": 32.408424772301395, |
| "mean_completion_seconds": 3.9495902963293097, |
| "mean_tpot_ms": 30.856174190072732, |
| "peak_observed_gpu_memory_mib": 23522, |
| "runs": [ |
| { |
| "generated_tokens": 2048, |
| "wall_seconds": 2.9401068949955516, |
| "output_tok_s": 696.5733128567418, |
| "gpu_memory_mib_after": 23430 |
| }, |
| { |
| "generated_tokens": 2048, |
| "wall_seconds": 3.787013527995441, |
| "output_tok_s": 540.7955331715059, |
| "gpu_memory_mib_after": 23430 |
| }, |
| { |
| "generated_tokens": 2048, |
| "wall_seconds": 5.121650465996936, |
| "output_tok_s": 399.8710988961161, |
| "gpu_memory_mib_after": 23522 |
| } |
| ] |
| }, |
| { |
| "workload": "text", |
| "concurrency": 32, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 12288, |
| "aggregate_output_tok_s": 1948.2966906600047, |
| "per_request_output_tok_s": 60.88427158312515, |
| "mean_completion_seconds": 2.1023492056604787, |
| "mean_tpot_ms": 16.42460316922249, |
| "peak_observed_gpu_memory_mib": 23914, |
| "runs": [ |
| { |
| "generated_tokens": 4096, |
| "wall_seconds": 2.0827834619558416, |
| "output_tok_s": 1966.5990607365604, |
| "gpu_memory_mib_after": 23522 |
| }, |
| { |
| "generated_tokens": 4096, |
| "wall_seconds": 2.070148140017409, |
| "output_tok_s": 1978.602362227833, |
| "gpu_memory_mib_after": 23914 |
| }, |
| { |
| "generated_tokens": 4096, |
| "wall_seconds": 2.154116015008185, |
| "output_tok_s": 1901.4760446801824, |
| "gpu_memory_mib_after": 23914 |
| } |
| ] |
| }, |
| { |
| "workload": "one_image_per_request", |
| "concurrency": 32, |
| "repeats": 3, |
| "output_tokens_per_request": 128, |
| "total_verified_output_tokens": 12288, |
| "aggregate_output_tok_s": 461.6294772584143, |
| "per_request_output_tok_s": 14.425921164325446, |
| "mean_completion_seconds": 8.872916921003101, |
| "mean_tpot_ms": 69.31966344533672, |
| "peak_observed_gpu_memory_mib": 23914, |
| "runs": [ |
| { |
| "generated_tokens": 4096, |
| "wall_seconds": 8.780865803011693, |
| "output_tok_s": 466.46880750587707, |
| "gpu_memory_mib_after": 23914 |
| }, |
| { |
| "generated_tokens": 4096, |
| "wall_seconds": 8.837103814992588, |
| "output_tok_s": 463.5002695171388, |
| "gpu_memory_mib_after": 23914 |
| }, |
| { |
| "generated_tokens": 4096, |
| "wall_seconds": 9.000781145005021, |
| "output_tok_s": 455.07161367578334, |
| "gpu_memory_mib_after": 23914 |
| } |
| ] |
| } |
| ] |
| } |
|
|
