{ "schema_version": 1, "run_id": "20260913-205443-614946", "maximize": "output_tokens_per_second", "execution_mode": "sequential", "status": "completed_with_failures", "started_at": "2026-09-13T20:54:43.614965+00:00", "completed_at": "2026-09-13T21:09:37.446758+00:00", "duration_seconds": 893.831793, "trial_counts": { "completed": 7, "failed": 2, "interrupted": 0 }, "trials": [ { "trial_id": "baseline", "status": "completed", "failure": null, "benchmark_count": 8, "metrics": { "requests_per_second": { "average": 4.440249042565908 }, "throughput_tokens_per_second": { "average": 191.61531847055863 }, "total_tokens_per_second": { "average": 772.9553518454742 }, "ttft_ms": { "average": 1346.4367349843656, "median": 1596.019846812851, "p95": 1773.1966687822478, "p99": 1880.852297355841 }, "tpot_ms": { "average": 5.039373205236442, "median": 4.649440601459702, "p95": 6.563462977028086, "p99": 9.393960609429522 }, "itl_ms": { "average": 5.158024149309876, "median": 4.663633249947452, "p99": 16.433663622619846 }, "end_to_end_ms": { "average": 1580.870448656185, "median": 1814.8021570627861, "p95": 1989.5453593442198, "p99": 2096.197147268458 }, "total_time_seconds": { "average": 3.978179454250494 } }, "benchmarks": [ { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/baseline/attempts/001/repeats/001/short-chat/results.json", "repeat": 1, "elapsed_seconds": 8.776927304999845, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235523", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.623472234001383, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 6.098787626807246, "request_goodput": null, "output_throughput": 195.16120405783187, "total_token_throughput": "", "ttfts": [ 0.42395702700014226, 0.04451068500202382, 0.5697069739981089, 0.7187096959969494, 0.8711377670042566, 1.0140277730024536, 1.161577336002665, 1.3104403859979357, 1.0470208490005461, 1.0472929569950793, 1.0485800960013876, 1.0404932239980553, 1.0444222969963448, 1.0395346079967567, 1.0400018630025443, 1.0361805760039715 ], "max_output_tokens_per_s": 210.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 903.5996321249513, "median_ttft_ms": 1039.7682354996505, "std_ttft_ms": 307.6675060420228, "p50_ttft_ms": 1039.7682354996505, "p95_ttft_ms": 1198.7930985014827, "p99_ttft_ms": 1288.1109284986449, "mean_tpot_ms": 4.886566570554175, "median_tpot_ms": 4.425130790261919, "std_tpot_ms": 1.8118595518821783, "p50_tpot_ms": 4.425130790261919, "p95_tpot_ms": 6.399209370952562, "p99_tpot_ms": 10.798277100020053, "mean_itl_ms": 5.0389596548552475, "median_itl_ms": 4.473193002922926, "std_itl_ms": 3.564089297046419, "p50_itl_ms": 4.473193002922926, "p95_itl_ms": 5.784334993222728, "p99_itl_ms": 25.39229560206877, "mean_e2el_ms": 1055.0831958121307, "median_e2el_ms": 1176.219452998339, "std_e2el_ms": 270.0129509485938, "p50_e2el_ms": 1176.219452998339, "p95_e2el_ms": 1336.9572642513958, "p99_e2el_ms": 1426.8840384487703, "requests_per_second": { "average": 6.098787626807246 }, "output_tokens_per_second": { "average": 195.16120405783187 }, "total_tokens_per_second": { "average": 585.4836121734957 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1055.0831958121307, "median": 1176.219452998339, "p99": 1426.8840384487703 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/baseline/attempts/001/repeats/002/short-chat/results.json", "repeat": 2, "elapsed_seconds": 8.913840654000524, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235532", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.7966449450032087, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 5.721140979510948, "request_goodput": null, "output_throughput": 183.07651134435034, "total_token_throughput": "", "ttfts": [ 0.04285283700301079, 0.4781834009991144, 0.681513637995522, 0.8532786349969683, 1.0189234700010275, 1.3425323310002568, 1.1845137839991366, 1.4858502030037926, 1.166356301997439, 1.101818554001511, 1.079139010005747, 1.0675376090002828, 1.0403984299991862, 1.0327826419961639, 1.0379568159987684, 1.0373018059981405 ], "max_output_tokens_per_s": 203.0, "max_concurrent_requests": 13, "rtfx": 0.0, "mean_ttft_ms": 978.1837167497542, "median_ttft_ms": 1039.1776229989773, "std_ttft_ms": 330.3365379354567, "p50_ttft_ms": 1039.1776229989773, "p95_ttft_ms": 1378.3617990011408, "p99_ttft_ms": 1464.352522203262, "mean_tpot_ms": 5.220979258102752, "median_tpot_ms": 4.513776306515454, "std_tpot_ms": 2.2193474292762994, "p50_tpot_ms": 4.513776306515454, "p95_tpot_ms": 8.058886008051735, "p99_tpot_ms": 12.499576375722102, "mean_itl_ms": 5.395018514582262, "median_itl_ms": 4.66417249845108, "std_itl_ms": 4.561385647446098, "p50_itl_ms": 4.66417249845108, "p95_itl_ms": 7.273197896211059, "p99_itl_ms": 33.022187081223805, "mean_e2el_ms": 1140.0340737509396, "median_e2el_ms": 1177.899394002452, "std_e2el_ms": 275.3329111145585, "p50_e2el_ms": 1177.899394002452, "p95_e2el_ms": 1512.2184642495995, "p99_e2el_ms": 1597.6987632508096, "requests_per_second": { "average": 5.721140979510948 }, "output_tokens_per_second": { "average": 183.07651134435034 }, "total_tokens_per_second": { "average": 549.2295340330511 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1140.0340737509396, "median": 1177.899394002452, "p99": 1597.6987632508096 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/baseline/attempts/001/repeats/003/short-chat/results.json", "repeat": 3, "elapsed_seconds": 8.921835840999847, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235541", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.6394274280028185, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 6.061920790186967, "request_goodput": null, "output_throughput": 193.98146528598295, "total_token_throughput": "", "ttfts": [ 0.028562259998579975, 0.42150033800135134, 0.5706503610053915, 0.7181204880034784, 1.1651781949985889, 0.8716665339961764, 1.0148062570006005, 1.3110276269944734, 1.0525595970029826, 1.0527448959983303, 1.0527994410003885, 1.0451392460017814, 1.0427730549999978, 1.0457294730003923, 1.0497605530035798, 1.0470561740003177 ], "max_output_tokens_per_s": 211.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 905.6296559379007, "median_ttft_ms": 1045.4343595010869, "std_ttft_ms": 312.1848242104956, "p50_ttft_ms": 1045.4343595010869, "p95_ttft_ms": 1201.64055299756, "p99_ttft_ms": 1289.1502121950907, "mean_tpot_ms": 4.9423984334621744, "median_tpot_ms": 4.47934799998503, "std_tpot_ms": 1.9143385441040246, "p50_tpot_ms": 4.47934799998503, "p95_tpot_ms": 6.49212744364406, "p99_tpot_ms": 11.179364275824717, "mean_itl_ms": 5.096533492760802, "median_itl_ms": 4.530316000455059, "std_itl_ms": 4.972773475044159, "p50_itl_ms": 4.530316000455059, "p95_itl_ms": 5.802618004963733, "p99_itl_ms": 20.38991719891778, "mean_e2el_ms": 1058.844007375228, "median_e2el_ms": 1184.5615455022198, "std_e2el_ms": 272.1612072832017, "p50_e2el_ms": 1184.5615455022198, "p95_e2el_ms": 1338.3068879993516, "p99_e2el_ms": 1428.6871223986964, "requests_per_second": { "average": 6.061920790186967 }, "output_tokens_per_second": { "average": 193.98146528598295 }, "total_tokens_per_second": { "average": 581.9443958579488 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1058.844007375228, "median": 1184.5615455022198, "p99": 1428.6871223986964 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/baseline/attempts/001/repeats/004/short-chat/results.json", "repeat": 4, "elapsed_seconds": 9.56328499600204, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235550", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 3.042805233002582, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 5.258305666909711, "request_goodput": null, "output_throughput": 168.26578134111074, "total_token_throughput": "", "ttfts": [ 0.02613996600121027, 0.8296127540015732, 0.3892902250008774, 1.0129026070062537, 0.6338503180013504, 1.1754765490040882, 1.3472542470044573, 1.5232661310001276, 1.3155866719971527, 1.2411482499956037, 1.2131225019984413, 1.1952597630006494, 1.2047895270006848, 1.201920726998651, 1.2017613549978705, 1.207303695002338 ], "max_output_tokens_per_s": 183.0, "max_concurrent_requests": 13, "rtfx": 0.0, "mean_ttft_ms": 1044.917830500708, "median_ttft_ms": 1201.8410409982607, "std_ttft_ms": 377.9240222439532, "p50_ttft_ms": 1201.8410409982607, "p95_ttft_ms": 1391.2572180033749, "p99_ttft_ms": 1496.864348400777, "mean_tpot_ms": 5.711755302399979, "median_tpot_ms": 5.178266048343146, "std_tpot_ms": 1.553785322714281, "p50_tpot_ms": 5.178266048343146, "p95_tpot_ms": 8.35385309684141, "p99_tpot_ms": 10.709281303277221, "mean_itl_ms": 5.889883338899223, "median_itl_ms": 5.247916000371333, "std_itl_ms": 2.755429548233552, "p50_itl_ms": 5.247916000371333, "p95_itl_ms": 8.216201000323053, "p99_itl_ms": 20.26060619682535, "mean_e2el_ms": 1221.9822448751074, "median_e2el_ms": 1361.5556955010106, "std_e2el_ms": 336.87792103582916, "p50_e2el_ms": 1361.5556955010106, "p95_e2el_ms": 1554.145095500644, "p99_e2el_ms": 1654.3710023008316, "requests_per_second": { "average": 5.258305666909711 }, "output_tokens_per_second": { "average": 168.26578134111074 }, "total_tokens_per_second": { "average": 504.79734402333224 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1221.9822448751074, "median": 1361.5556955010106, "p99": 1654.3710023008316 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/baseline/attempts/001/repeats/001/longer-prompt/results.json", "repeat": 1, "elapsed_seconds": 11.766263288001937, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235614", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 5.569405582995387, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 2.8728380006748835, "request_goodput": null, "output_throughput": 183.86163204319254, "total_token_throughput": "", "ttfts": [ 0.048843800999748055, 0.6963653200000408, 1.3613970969963702, 1.0349977020014194, 1.6926306809982634, 2.0163684700019076, 2.347111640003277, 2.654207924999355, 2.2819521659985185, 2.2610513210020144, 2.269123555997794, 2.2596722899979795, 2.244481131005159, 2.2337055710013374, 2.274074307999399, 2.2978764439976658 ], "max_output_tokens_per_s": 198.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 1873.3662139375156, "median_ttft_ms": 2252.0767105015693, "std_ttft_ms": 698.1178217785232, "p50_ttft_ms": 2252.0767105015693, "p95_ttft_ms": 2423.8857112522965, "p99_ttft_ms": 2608.143482249943, "mean_tpot_ms": 5.3029800337248565, "median_tpot_ms": 4.9901343095010064, "std_tpot_ms": 1.251760768199272, "p50_tpot_ms": 4.9901343095010064, "p95_tpot_ms": 6.551825428542789, "p99_tpot_ms": 9.39479266665922, "mean_itl_ms": 5.383088390743779, "median_itl_ms": 4.839831999561284, "std_itl_ms": 5.652736034452867, "p50_itl_ms": 4.839831999561284, "p95_itl_ms": 7.0334259973606095, "p99_itl_ms": 9.996185843483547, "mean_e2el_ms": 2207.4539560621815, "median_e2el_ms": 2563.8645975013787, "std_e2el_ms": 644.9311356110081, "p50_e2el_ms": 2563.8645975013787, "p95_e2el_ms": 2721.6993577494577, "p99_e2el_ms": 2908.0415107462613, "requests_per_second": { "average": 2.8728380006748835 }, "output_tokens_per_second": { "average": 183.86163204319254 }, "total_tokens_per_second": { "average": 919.3081602159626 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 2207.4539560621815, "median": 2563.8645975013787, "p99": 2908.0415107462613 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/baseline/attempts/001/repeats/002/longer-prompt/results.json", "repeat": 2, "elapsed_seconds": 11.448642527997436, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235625", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 4.91661816299893, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.2542693920002677, "request_goodput": null, "output_throughput": 208.27324108801713, "total_token_throughput": "", "ttfts": [ 0.028628286003367975, 0.48218945999542484, 0.7677247310057282, 1.0642265939968638, 1.3659639069956029, 1.6632926109959953, 1.95808783499524, 2.2472684040039894, 2.070958055999654, 2.0721031470020534, 2.090979783999501, 2.09979028700036, 2.0969477699982235, 2.098269165995589, 2.098939834999328, 2.097437026000989 ], "max_output_tokens_per_s": 216.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 1643.9254311867444, "median_ttft_ms": 2071.5306015008537, "std_ttft_ms": 670.8522528039375, "p50_ttft_ms": 2071.5306015008537, "p95_ttft_ms": 2136.6598162512673, "p99_ttft_ms": 2225.146686453445, "mean_tpot_ms": 4.687403469240905, "median_tpot_ms": 4.507854674542729, "std_tpot_ms": 0.6169950376891856, "p50_tpot_ms": 4.507854674542729, "p95_tpot_ms": 5.343857682520238, "p99_tpot_ms": 6.697935320652692, "mean_itl_ms": 4.758213348417638, "median_itl_ms": 4.491218998737168, "std_itl_ms": 3.8372868387267025, "p50_itl_ms": 4.491218998737168, "p95_itl_ms": 5.810285196639597, "p99_itl_ms": 7.077269116707612, "mean_e2el_ms": 1939.2318497489214, "median_e2el_ms": 2359.975552997639, "std_e2el_ms": 648.8822173835924, "p50_e2el_ms": 2359.975552997639, "p95_e2el_ms": 2426.8113952493877, "p99_e2el_ms": 2510.1228718544007, "requests_per_second": { "average": 3.2542693920002677 }, "output_tokens_per_second": { "average": 208.27324108801713 }, "total_tokens_per_second": { "average": 1041.3662054400857 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1939.2318497489214, "median": 2359.975552997639, "p99": 2510.1228718544007 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/baseline/attempts/001/repeats/003/longer-prompt/results.json", "repeat": 3, "elapsed_seconds": 11.412307805003366, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235637", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 5.007630688996869, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.1951238007939295, "request_goodput": null, "output_throughput": 204.4879232508115, "total_token_throughput": "", "ttfts": [ 0.031211353998514824, 0.5960343970000395, 0.9022867830062751, 1.2020089229990845, 2.0835528789975797, 1.4902997370008961, 1.790529316996981, 2.3637015529966448, 2.0673730810012785, 2.0439635169968824, 2.0442959200008772, 2.0676904150022892, 2.0519892680022167, 2.065162409002369, 2.077226334004081, 2.08804082700226 ], "max_output_tokens_per_s": 220.0, "max_concurrent_requests": 11, "rtfx": 0.0, "mean_ttft_ms": 1685.3354196255168, "median_ttft_ms": 2048.142594001547, "std_ttft_ms": 641.35617210578, "p50_ttft_ms": 2048.142594001547, "p95_ttft_ms": 2156.956008500856, "p99_ttft_ms": 2322.3524440974866, "mean_tpot_ms": 4.779947157744789, "median_tpot_ms": 4.523479301583609, "std_tpot_ms": 1.0389539532073924, "p50_tpot_ms": 4.523479301583609, "p95_tpot_ms": 5.762240551583033, "p99_tpot_ms": 8.167796161063643, "mean_itl_ms": 4.857046651222258, "median_itl_ms": 4.446356499101967, "std_itl_ms": 3.990733173031452, "p50_itl_ms": 4.446356499101967, "p95_itl_ms": 5.901613051173623, "p99_itl_ms": 8.163016457183403, "mean_e2el_ms": 1986.4720905634385, "median_e2el_ms": 2342.4975125017227, "std_e2el_ms": 597.6024031860154, "p50_e2el_ms": 2342.4975125017227, "p95_e2el_ms": 2434.529652753554, "p99_e2el_ms": 2598.7153945476166, "requests_per_second": { "average": 3.1951238007939295 }, "output_tokens_per_second": { "average": 204.4879232508115 }, "total_tokens_per_second": { "average": 1022.4396162540575 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1986.4720905634385, "median": 2342.4975125017227, "p99": 2598.7153945476166 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/baseline/attempts/001/repeats/004/longer-prompt/results.json", "repeat": 4, "elapsed_seconds": 11.496019571000943, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235648", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 5.229431359002774, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.059606083643312, "request_goodput": null, "output_throughput": 195.81478935317196, "total_token_throughput": "", "ttfts": [ 0.2426799749955535, 1.1897016590010026, 2.081059998003184, 0.6144731250024051, 0.9039233069997863, 1.4808468089977396, 1.769225109994295, 2.366367406997597, 2.065047691001382, 2.0692976970021846, 2.071077521999541, 2.0757333949950407, 2.139268680999521, 2.1863618700008374, 2.2542750389984576, 2.2752363920008065 ], "max_output_tokens_per_s": 217.0, "max_concurrent_requests": 11, "rtfx": 0.0, "mean_ttft_ms": 1736.5359798118334, "median_ttft_ms": 2070.187609500863, "std_ttft_ms": 634.0911049496651, "p50_ttft_ms": 2070.187609500863, "p95_ttft_ms": 2298.019145750004, "p99_ttft_ms": 2352.6977547480783, "mean_tpot_ms": 4.782955416661905, "median_tpot_ms": 4.577535380944721, "std_tpot_ms": 0.4534488920754421, "p50_tpot_ms": 4.577535380944721, "p95_tpot_ms": 5.545704234088863, "p99_tpot_ms": 5.704661672216533, "mean_itl_ms": 4.845449802997796, "median_itl_ms": 4.616060999978799, "std_itl_ms": 2.1166133597620185, "p50_itl_ms": 4.616060999978799, "p95_itl_ms": 6.225794098281766, "p99_itl_ms": 7.167831484548513, "mean_e2el_ms": 2037.8621710615334, "median_e2el_ms": 2351.843505497527, "std_e2el_ms": 633.7300120833249, "p50_e2el_ms": 2351.843505497527, "p95_e2el_ms": 2591.6947570003686, "p99_e2el_ms": 2645.0564746002783, "requests_per_second": { "average": 3.059606083643312 }, "output_tokens_per_second": { "average": 195.81478935317196 }, "total_tokens_per_second": { "average": 979.0739467658598 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 2037.8621710615334, "median": 2351.843505497527, "p99": 2645.0564746002783 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] } ], "execution": { "mode": "sequential" } }, { "trial_id": "trial-0001", "status": "completed", "failure": null, "benchmark_count": 8, "metrics": { "requests_per_second": { "average": 4.483297055184135 }, "throughput_tokens_per_second": { "average": 191.7421011939862 }, "total_tokens_per_second": { "average": 768.3326852943342 }, "ttft_ms": { "average": 1344.7469420469247, "median": 1646.9772212512908, "p95": 1763.8610329986477, "p99": 1835.944086500058 }, "tpot_ms": { "average": 5.076305310701006, "median": 4.7477207104881884, "p95": 6.680251803670813, "p99": 8.071265940051592 }, "itl_ms": { "average": 5.178244993952775, "median": 4.7751813131071685, "p99": 11.670120199705693 }, "end_to_end_ms": { "average": 1582.7460517341478, "median": 1871.477548499115, "p95": 1996.9882334050908, "p99": 2067.6559178807565 }, "total_time_seconds": { "average": 4.0337422435022745 } }, "benchmarks": [ { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0001/attempts/001/repeats/001/short-chat/results.json", "repeat": 1, "elapsed_seconds": 8.790463592005835, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235722", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.4302008250015206, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 6.583818026639831, "request_goodput": null, "output_throughput": 210.68217685247458, "total_token_throughput": "", "ttfts": [ 0.01956779900501715, 0.17680020500120008, 0.639073357997404, 0.3398486750011216, 0.49284180300310254, 0.933256614000129, 0.7870622500049649, 1.0775813799991738, 1.0571199779951712, 1.0408673490019282, 1.028128274003393, 1.02555808499892, 1.0188737029966433, 1.0161441010059207, 1.0348016830030247, 1.0706708420038922 ], "max_output_tokens_per_s": 215.0, "max_concurrent_requests": 14, "rtfx": 0.0, "mean_ttft_ms": 797.3872561888129, "median_ttft_ms": 1017.508902001282, "std_ttft_ms": 342.26883862821586, "p50_ttft_ms": 1017.508902001282, "p95_ttft_ms": 1072.3984765027126, "p99_ttft_ms": 1076.5447992998816, "mean_tpot_ms": 4.5366874717640915, "median_tpot_ms": 4.387877564553374, "std_tpot_ms": 0.3192279390847849, "p50_tpot_ms": 4.387877564553374, "p95_tpot_ms": 5.099838169342313, "p99_tpot_ms": 5.338095401689755, "mean_itl_ms": 4.668466041492071, "median_itl_ms": 4.57168400316732, "std_itl_ms": 0.7160694701978784, "p50_itl_ms": 4.57168400316732, "p95_itl_ms": 5.779076548424199, "p99_itl_ms": 7.264357826206833, "mean_e2el_ms": 938.0245678134997, "median_e2el_ms": 1154.5668704966374, "std_e2el_ms": 340.7478164408282, "p50_e2el_ms": 1154.5668704966374, "p95_e2el_ms": 1212.456277999081, "p99_e2el_ms": 1216.7088428028364, "requests_per_second": { "average": 6.583818026639831 }, "output_tokens_per_second": { "average": 210.68217685247458 }, "total_tokens_per_second": { "average": 632.0465305574238 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 938.0245678134997, "median": 1154.5668704966374, "p99": 1216.7088428028364 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0001/attempts/001/repeats/002/short-chat/results.json", "repeat": 2, "elapsed_seconds": 9.09055348399852, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235732", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.7958256780038937, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 5.72281745814115, "request_goodput": null, "output_throughput": 183.1301586605168, "total_token_throughput": "", "ttfts": [ 0.07891567100159591, 0.47162526800093474, 0.9166377930014278, 0.6168051349959569, 0.7610912069940241, 1.0710009719987283, 1.2171523439974408, 1.365271588001633, 1.0648984759973246, 1.073095801002637, 1.0696867810038384, 1.059554452993325, 1.0845503700038535, 1.1048030180027126, 1.1521869520001928, 1.1432358900055988 ], "max_output_tokens_per_s": 208.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 953.1569824375765, "median_ttft_ms": 1070.3438765012834, "std_ttft_ms": 313.1213927618803, "p50_ttft_ms": 1070.3438765012834, "p95_ttft_ms": 1254.1821549984888, "p99_ttft_ms": 1343.0537014010042, "mean_tpot_ms": 5.141384078577234, "median_tpot_ms": 4.49754588701199, "std_tpot_ms": 1.9132081384204858, "p50_tpot_ms": 4.49754588701199, "p95_tpot_ms": 7.55522739505508, "p99_tpot_ms": 11.374885014459233, "mean_itl_ms": 5.268862481387274, "median_itl_ms": 4.612064003595151, "std_itl_ms": 9.447253290587485, "p50_itl_ms": 4.612064003595151, "p95_itl_ms": 6.556490347429643, "p99_itl_ms": 7.800399453844877, "mean_e2el_ms": 1112.5398888734708, "median_e2el_ms": 1206.7526764985814, "std_e2el_ms": 277.79026767085554, "p50_e2el_ms": 1206.7526764985814, "p95_e2el_ms": 1394.1023502484313, "p99_e2el_ms": 1489.494855648081, "requests_per_second": { "average": 5.72281745814115 }, "output_tokens_per_second": { "average": 183.1301586605168 }, "total_tokens_per_second": { "average": 549.3904759815504 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1112.5398888734708, "median": 1206.7526764985814, "p99": 1489.494855648081 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0001/attempts/001/repeats/003/short-chat/results.json", "repeat": 3, "elapsed_seconds": 8.4955614329956, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235740", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.3566001700019115, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 6.789441927260415, "request_goodput": null, "output_throughput": 217.26214167233329, "total_token_throughput": "", "ttfts": [ 0.01661810199584579, 0.1640243720030412, 0.31696138600091217, 0.4599505649966886, 0.60734033400513, 1.0431886549995397, 0.7521909009956289, 0.9000038649974158, 1.0371328779947362, 1.0291343650023919, 1.0292019109983812, 1.0243866599994362, 1.024101315997541, 1.0231670330031193, 1.0287673490020097, 1.0298150739981793 ], "max_output_tokens_per_s": 217.0, "max_concurrent_requests": 14, "rtfx": 0.0, "mean_ttft_ms": 780.3740478743748, "median_ttft_ms": 1023.6341745003301, "std_ttft_ms": 343.3553246202186, "p50_ttft_ms": 1023.6341745003301, "p95_ttft_ms": 1038.646822245937, "p99_ttft_ms": 1042.2802884488192, "mean_tpot_ms": 4.405717699624333, "median_tpot_ms": 4.407556838780311, "std_tpot_ms": 0.12137543225172445, "p50_tpot_ms": 4.407556838780311, "p95_tpot_ms": 4.636866024215593, "p99_tpot_ms": 4.688900482259698, "mean_itl_ms": 4.496375537043751, "median_itl_ms": 4.4711424998240545, "std_itl_ms": 0.6780007785701893, "p50_itl_ms": 4.4711424998240545, "p95_itl_ms": 5.430552748293849, "p99_itl_ms": 6.4413490516017164, "mean_e2el_ms": 916.9512965627291, "median_e2el_ms": 1159.6318979973148, "std_e2el_ms": 342.9506988161674, "p50_e2el_ms": 1159.6318979973148, "p95_e2el_ms": 1176.6340032481821, "p99_e2el_ms": 1179.177396648447, "requests_per_second": { "average": 6.789441927260415 }, "output_tokens_per_second": { "average": 217.26214167233329 }, "total_tokens_per_second": { "average": 651.7864250169998 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 916.9512965627291, "median": 1159.6318979973148, "p99": 1179.177396648447 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0001/attempts/001/repeats/004/short-chat/results.json", "repeat": 4, "elapsed_seconds": 9.663830045996292, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235750", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 3.4034224420029204, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 4.70115017240821, "request_goodput": null, "output_throughput": 150.4368055170627, "total_token_throughput": "", "ttfts": [ 0.028500282001914456, 0.7951074730008258, 0.42936268499761354, 0.6182143109981553, 0.9639256070004194, 1.1460109499967075, 1.3319172549963696, 1.5571231080029975, 1.399098440000671, 1.4132998700006283, 1.420268171998032, 1.4516083580019767, 1.4692425089961034, 1.464476689005096, 1.459620025001641, 1.423893009996391 ], "max_output_tokens_per_s": 158.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 1148.2292964997214, "median_ttft_ms": 1406.1991550006496, "std_ttft_ms": 439.19953251423556, "p50_ttft_ms": 1406.1991550006496, "p95_ttft_ms": 1491.212658747827, "p99_ttft_ms": 1543.9410181519634, "mean_tpot_ms": 6.368241971769396, "median_tpot_ms": 5.767696290398993, "std_tpot_ms": 1.7314842757969178, "p50_tpot_ms": 5.767696290398993, "p95_tpot_ms": 9.009796967685332, "p99_tpot_ms": 11.833524787031028, "mean_itl_ms": 6.5805243000189275, "median_itl_ms": 5.842851500347024, "std_itl_ms": 4.915030851910017, "p50_itl_ms": 5.842851500347024, "p95_itl_ms": 9.08141255276859, "p99_itl_ms": 12.745136450684988, "mean_e2el_ms": 1345.6447976245727, "median_e2el_ms": 1587.502060996485, "std_e2el_ms": 418.24739391175433, "p50_e2el_ms": 1587.502060996485, "p95_e2el_ms": 1704.8259355033224, "p99_e2el_ms": 1780.9281175017531, "requests_per_second": { "average": 4.70115017240821 }, "output_tokens_per_second": { "average": 150.4368055170627 }, "total_tokens_per_second": { "average": 451.31041655118815 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1345.6447976245727, "median": 1587.502060996485, "p99": 1780.9281175017531 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0001/attempts/001/repeats/001/longer-prompt/results.json", "repeat": 1, "elapsed_seconds": 11.078219220005849, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235812", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 4.952123183000367, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.2309373997247786, "request_goodput": null, "output_throughput": 206.77999358238583, "total_token_throughput": "", "ttfts": [ 0.02044070199917769, 0.5283677190018352, 0.8185863239996252, 1.1188345929986099, 1.411674173999927, 1.705732763002743, 2.0013634909992106, 2.3044674530028715, 2.0850152959974366, 2.086818379997567, 2.082630925004196, 2.0850375769950915, 2.0954013700029464, 2.095045702000789, 2.085579589998815, 2.0748517680040095 ], "max_output_tokens_per_s": 214.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 1662.4904891878032, "median_ttft_ms": 2078.7413465041027, "std_ttft_ms": 662.346842647489, "p50_ttft_ms": 2078.7413465041027, "p95_ttft_ms": 2147.6678907529276, "p99_ttft_ms": 2273.1075405528823, "mean_tpot_ms": 4.733772454348207, "median_tpot_ms": 4.509555992020371, "std_tpot_ms": 0.8156405318543029, "p50_tpot_ms": 4.509555992020371, "p95_tpot_ms": 5.481881277788373, "p99_tpot_ms": 7.403435087297341, "mean_itl_ms": 4.795623797982996, "median_itl_ms": 4.498365000472404, "std_itl_ms": 1.6835699625650784, "p50_itl_ms": 4.498365000472404, "p95_itl_ms": 6.03784749910119, "p99_itl_ms": 14.431233514333135, "mean_e2el_ms": 1960.7181538117402, "median_e2el_ms": 2362.1258650018717, "std_e2el_ms": 631.3098466829164, "p50_e2el_ms": 2362.1258650018717, "p95_e2el_ms": 2432.8177127463277, "p99_e2el_ms": 2559.6874793467578, "requests_per_second": { "average": 3.2309373997247786 }, "output_tokens_per_second": { "average": 206.77999358238583 }, "total_tokens_per_second": { "average": 1033.8999679119293 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1960.7181538117402, "median": 2362.1258650018717, "p99": 2559.6874793467578 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0001/attempts/001/repeats/002/longer-prompt/results.json", "repeat": 2, "elapsed_seconds": 11.709677316001034, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235824", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 5.318329530004121, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.0084634488580857, "request_goodput": null, "output_throughput": 192.54166072691748, "total_token_throughput": "", "ttfts": [ 0.06892771199636627, 0.6355955719991471, 1.5569055910018506, 0.9452412989994627, 1.2575953769992338, 1.8741974329968798, 2.168451534998894, 2.47643932900246, 2.147883976002049, 2.147349301005306, 2.139411887001188, 2.1846380650022184, 2.173867715006054, 2.2467699330009054, 2.2376611590007087, 2.268619028000103 ], "max_output_tokens_per_s": 211.0, "max_concurrent_requests": 11, "rtfx": 0.0, "mean_ttft_ms": 1783.0971820008017, "median_ttft_ms": 2147.6166385036777, "std_ttft_ms": 674.8135348448568, "p50_ttft_ms": 2147.6166385036777, "p95_ttft_ms": 2320.574103250692, "p99_ttft_ms": 2445.2662838521064, "mean_tpot_ms": 5.039706563467107, "median_tpot_ms": 4.7348496586819815, "std_tpot_ms": 1.0171403740395841, "p50_tpot_ms": 4.7348496586819815, "p95_tpot_ms": 6.443056753957377, "p99_tpot_ms": 8.330527173085201, "mean_itl_ms": 5.115838427974879, "median_itl_ms": 4.649324997444637, "std_itl_ms": 3.1967935992933283, "p50_itl_ms": 4.649324997444637, "p95_itl_ms": 6.600731199432629, "p99_itl_ms": 10.42368963564522, "mean_e2el_ms": 2100.5986954992295, "median_e2el_ms": 2457.124501499493, "std_e2el_ms": 638.046085795458, "p50_e2el_ms": 2457.124501499493, "p95_e2el_ms": 2608.5581402494427, "p99_e2el_ms": 2727.704775250095, "requests_per_second": { "average": 3.0084634488580857 }, "output_tokens_per_second": { "average": 192.54166072691748 }, "total_tokens_per_second": { "average": 962.7083036345874 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 2100.5986954992295, "median": 2457.124501499493, "p99": 2727.704775250095 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0001/attempts/001/repeats/003/longer-prompt/results.json", "repeat": 3, "elapsed_seconds": 11.920072181004798, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235836", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 5.818300496997836, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 2.74994390686005, "request_goodput": null, "output_throughput": 175.9964100390432, "total_token_throughput": "", "ttfts": [ 0.033097522995376494, 0.6210782890047994, 1.6146867260031286, 0.9150717379961861, 2.3255393489962444, 1.9961806360006449, 1.2585931610010448, 2.626220801997988, 2.348308103995805, 2.3826921689978917, 2.3483745799967437, 2.3035242629994173, 2.2542700659978436, 2.5161776339955395, 2.5156468600034714, 2.515293472999474 ], "max_output_tokens_per_s": 200.0, "max_concurrent_requests": 11, "rtfx": 0.0, "mean_ttft_ms": 1910.92221081135, "median_ttft_ms": 2314.531805997831, "std_ttft_ms": 764.8656263573087, "p50_ttft_ms": 2314.531805997831, "p95_ttft_ms": 2543.6884259961516, "p99_ttft_ms": 2609.7143267976207, "mean_tpot_ms": 5.558378182560397, "median_tpot_ms": 5.048367238140642, "std_tpot_ms": 1.4281903844251278, "p50_tpot_ms": 5.048367238140642, "p95_tpot_ms": 9.188461539740693, "p99_tpot_ms": 9.253633844433816, "mean_itl_ms": 5.625350001993874, "median_itl_ms": 4.937507001159247, "std_itl_ms": 3.996321544135515, "p50_itl_ms": 4.937507001159247, "p95_itl_ms": 7.712862994594616, "p99_itl_ms": 25.656176645497823, "mean_e2el_ms": 2261.100036312655, "median_e2el_ms": 2636.340338001901, "std_e2el_ms": 735.6191634499137, "p50_e2el_ms": 2636.340338001901, "p95_e2el_ms": 2889.793301001191, "p99_e2el_ms": 2934.4753106008284, "requests_per_second": { "average": 2.74994390686005 }, "output_tokens_per_second": { "average": 175.9964100390432 }, "total_tokens_per_second": { "average": 879.9820501952161 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 2261.100036312655, "median": 2636.340338001901, "p99": 2934.4753106008284 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0001/attempts/001/repeats/004/longer-prompt/results.json", "repeat": 4, "elapsed_seconds": 11.427983067995228, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235847", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 5.195135623005626, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.079804101580559, "request_goodput": null, "output_throughput": 197.10746250115577, "total_token_throughput": "", "ttfts": [ 0.16562724499817705, 0.5807817050008452, 0.8719994259954547, 1.1669638510065852, 1.4582316870000795, 1.748344574996736, 2.0702034319983795, 2.38142648399662, 2.1187951890024124, 2.1237613190023694, 2.129836455002078, 2.158466098997451, 2.150527083002089, 2.115688552999927, 2.1202212250063894, 2.19621481399372 ], "max_output_tokens_per_s": 212.0, "max_concurrent_requests": 11, "rtfx": 0.0, "mean_ttft_ms": 1722.318071374957, "median_ttft_ms": 2117.24187100117, "std_ttft_ms": 650.9307898654081, "p50_ttft_ms": 2117.24187100117, "p95_ttft_ms": 2242.517731494445, "p99_ttft_ms": 2353.644733496185, "mean_tpot_ms": 4.826554063497281, "median_tpot_ms": 4.628316214317844, "std_tpot_ms": 0.5539994344007793, "p50_tpot_ms": 4.628316214317844, "p95_tpot_ms": 6.026886301581741, "p99_tpot_ms": 6.347125730156669, "mean_itl_ms": 4.874919363728422, "median_itl_ms": 4.618511498847511, "std_itl_ms": 1.4191750699463999, "p50_itl_ms": 4.618511498847511, "p95_itl_ms": 6.452435499886633, "p99_itl_ms": 8.598619019830945, "mean_e2el_ms": 2026.3909773752857, "median_e2el_ms": 2407.7761775006365, "std_e2el_ms": 641.7399570528088, "p50_e2el_ms": 2407.7761775006365, "p95_e2el_ms": 2556.718146244748, "p99_e2el_ms": 2653.0705652472534, "requests_per_second": { "average": 3.079804101580559 }, "output_tokens_per_second": { "average": 197.10746250115577 }, "total_tokens_per_second": { "average": 985.5373125057788 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 2026.3909773752857, "median": 2407.7761775006365, "p99": 2653.0705652472534 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] } ], "execution": { "mode": "sequential" } }, { "trial_id": "trial-0002", "status": "completed", "failure": null, "benchmark_count": 8, "metrics": { "requests_per_second": { "average": 15.127008065763016 }, "throughput_tokens_per_second": { "average": 659.8904632400345 }, "total_tokens_per_second": { "average": 2682.9762102625755 }, "ttft_ms": { "average": 242.6661262112475, "median": 249.0476333132392, "p95": 433.085006125566, "p99": 433.25910122630376 }, "tpot_ms": { "average": 5.835140147693257, "median": 5.043231608122511, "p95": 8.579934747880024, "p99": 8.7327428984999 }, "itl_ms": { "average": 5.857378199575713, "median": 4.88983543618815, "p99": 18.977385827311082 }, "end_to_end_ms": { "average": 511.3909182658176, "median": 491.69054474987206, "p95": 658.549299782635, "p99": 658.767790758202 }, "total_time_seconds": { "average": 1.1476638875028584 } }, "benchmarks": [ { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0002/attempts/001/repeats/001/short-chat/results.json", "repeat": 1, "elapsed_seconds": 6.984428415998991, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235918", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8443845140063786, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 18.94871321607295, "request_goodput": null, "output_throughput": 606.3588229143344, "total_token_throughput": "", "ttfts": [ 0.02461031799612101, 0.0414386900010868, 0.04100427999946987, 0.04100160999951186, 0.35378899500210537, 0.35853562600095756, 0.3585098320036195, 0.3584162490005838, 0.18719762000546325, 0.18572414200025378, 0.18526200400083326, 0.18482877399947029, 0.1695625900028972, 0.16930090300593292, 0.16909407500497764, 0.16899921499862103 ], "max_output_tokens_per_s": 512.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 187.32968268886907, "median_ttft_ms": 177.19568200118374, "std_ttft_ms": 113.88058741757798, "p50_ttft_ms": 177.19568200118374, "p95_ttft_ms": 358.51628050295403, "p99_ttft_ms": 358.53175690135686, "mean_tpot_ms": 6.262266393121301, "median_tpot_ms": 5.160053451676973, "std_tpot_ms": 2.1807430682424513, "p50_tpot_ms": 5.160053451676973, "p95_tpot_ms": 10.026700459650696, "p99_tpot_ms": 10.231033227459957, "mean_itl_ms": 6.262270211693483, "median_itl_ms": 5.031363500165753, "std_itl_ms": 3.66752719139393, "p50_itl_ms": 5.031363500165753, "p95_itl_ms": 15.86832724569831, "p99_itl_ms": 20.04173949972028, "mean_e2el_ms": 381.4599408756294, "median_e2el_ms": 340.1934044995869, "std_e2el_ms": 81.84111813287988, "p50_e2el_ms": 340.1934044995869, "p95_e2el_ms": 523.5157820006862, "p99_e2el_ms": 523.7874883987388, "requests_per_second": { "average": 18.94871321607295 }, "output_tokens_per_second": { "average": 606.3588229143344 }, "total_tokens_per_second": { "average": 1819.076468743003 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 381.4599408756294, "median": 340.1934044995869, "p99": 523.7874883987388 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0002/attempts/001/repeats/002/short-chat/results.json", "repeat": 2, "elapsed_seconds": 7.087848715993459, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235925", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8439788710020366, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 18.957820568426754, "request_goodput": null, "output_throughput": 606.6502581896561, "total_token_throughput": "", "ttfts": [ 0.032989290994009934, 0.04876944799616467, 0.0482631949998904, 0.38476270400133217, 0.04816462800226873, 0.37965701899520354, 0.3841784190008184, 0.38366464199498296, 0.16521186699537793, 0.1649080170027446, 0.16487995899660746, 0.16428224999981467, 0.16573759699531365, 0.1660122479952406, 0.16586927300522802, 0.165784420001728 ], "max_output_tokens_per_s": 509.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 189.57093606104536, "median_ttft_ms": 165.4747319953458, "std_ttft_ms": 122.16404682270787, "p50_ttft_ms": 165.4747319953458, "p95_ttft_ms": 384.32449025094684, "p99_ttft_ms": 384.6750612512551, "mean_tpot_ms": 6.218026135112239, "median_tpot_ms": 4.761450193449666, "std_tpot_ms": 2.548364504481632, "p50_tpot_ms": 4.761450193449666, "p95_tpot_ms": 10.638276919346554, "p99_tpot_ms": 10.833856132341035, "mean_itl_ms": 6.255867606513348, "median_itl_ms": 4.848683995078318, "std_itl_ms": 3.6993560609787006, "p50_itl_ms": 4.848683995078318, "p95_itl_ms": 14.771011797711248, "p99_itl_ms": 21.03118972183438, "mean_e2el_ms": 382.32974624952476, "median_e2el_ms": 343.87541449905257, "std_e2el_ms": 88.06558053822829, "p50_e2el_ms": 343.87541449905257, "p95_e2el_ms": 529.9917327520234, "p99_e2el_ms": 530.243405755391, "requests_per_second": { "average": 18.957820568426754 }, "output_tokens_per_second": { "average": 606.6502581896561 }, "total_tokens_per_second": { "average": 1819.9507745689684 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 382.32974624952476, "median": 343.87541449905257, "p99": 530.243405755391 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0002/attempts/001/repeats/003/short-chat/results.json", "repeat": 3, "elapsed_seconds": 7.063925679001841, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235932", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.9152150230002007, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 17.482230511852613, "request_goodput": null, "output_throughput": 559.4313763792836, "total_token_throughput": "", "ttfts": [ 0.04189894799492322, 0.025617598003009334, 0.041484828005195595, 0.04143439999461407, 0.39246294899930945, 0.39874815700022737, 0.39861289899999974, 0.3984294360052445, 0.16889617800188717, 0.1677687089977553, 0.16768380299618002, 0.1675856069996371, 0.19749805000174092, 0.19820747500489233, 0.19812447999720462, 0.197961760997714 ], "max_output_tokens_per_s": 509.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 200.15095487497092, "median_ttft_ms": 183.19711400181404, "std_ttft_ms": 128.73362250693094, "p50_ttft_ms": 183.19711400181404, "p95_ttft_ms": 398.64671350005665, "p99_ttft_ms": 398.7278683001932, "mean_tpot_ms": 6.78717919758891, "median_tpot_ms": 5.612294629005896, "std_tpot_ms": 2.595563027672871, "p50_tpot_ms": 5.612294629005896, "p95_tpot_ms": 11.241381895219561, "p99_tpot_ms": 11.438338804826696, "mean_itl_ms": 6.8284843063159535, "median_itl_ms": 5.21145300444914, "std_itl_ms": 3.6576341978906193, "p50_itl_ms": 5.21145300444914, "p95_itl_ms": 15.173177402175494, "p99_itl_ms": 18.97521547827637, "mean_e2el_ms": 410.5535100002271, "median_e2el_ms": 378.33138900168706, "std_e2el_ms": 78.37284669813879, "p50_e2el_ms": 378.33138900168706, "p95_e2el_ms": 544.8563822501455, "p99_e2el_ms": 545.0174108511419, "requests_per_second": { "average": 17.482230511852613 }, "output_tokens_per_second": { "average": 559.4313763792836 }, "total_tokens_per_second": { "average": 1678.2941291378509 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 410.5535100002271, "median": 378.33138900168706, "p99": 545.0174108511419 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0002/attempts/001/repeats/004/short-chat/results.json", "repeat": 4, "elapsed_seconds": 6.979001060994051, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235939", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.7383224290024373, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 21.67074894584732, "request_goodput": null, "output_throughput": 693.4639662671142, "total_token_throughput": "", "ttfts": [ 0.035843505000229925, 0.02564188499673037, 0.0353726609973819, 0.03497141499974532, 0.2813133950039628, 0.285759604004852, 0.28569341999536846, 0.28523103299812647, 0.16763240300497273, 0.1673229119987809, 0.1672135420012637, 0.16715813399787294, 0.1651006040046923, 0.16340060099901166, 0.16326798999944003, 0.16317051200167043 ], "max_output_tokens_per_s": 509.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 162.13085100025637, "median_ttft_ms": 166.1293690012826, "std_ttft_ms": 89.0385944052329, "p50_ttft_ms": 166.1293690012826, "p95_ttft_ms": 285.70996599773935, "p99_ttft_ms": 285.7496764034295, "mean_tpot_ms": 5.454787586703817, "median_tpot_ms": 4.697491725833501, "std_tpot_ms": 1.378510001771774, "p50_tpot_ms": 4.697491725833501, "p95_tpot_ms": 7.838332145301167, "p99_tpot_ms": 7.9239227646359085, "mean_itl_ms": 5.487984498984249, "median_itl_ms": 4.6831719955662265, "std_itl_ms": 3.2152078237452306, "p50_itl_ms": 4.6831719955662265, "p95_itl_ms": 11.13700679707108, "p99_itl_ms": 21.753838435106445, "mean_e2el_ms": 331.2292661880747, "median_e2el_ms": 310.80267099969205, "std_e2el_ms": 60.03099895218822, "p50_e2el_ms": 310.80267099969205, "p95_e2el_ms": 434.1694067497883, "p99_e2el_ms": 434.28619495425664, "requests_per_second": { "average": 21.67074894584732 }, "output_tokens_per_second": { "average": 693.4639662671142 }, "total_tokens_per_second": { "average": 2080.3918988013424 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 331.2292661880747, "median": 310.80267099969205, "p99": 434.28619495425664 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0002/attempts/001/repeats/001/longer-prompt/results.json", "repeat": 1, "elapsed_seconds": 7.508312466998177, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260913-235955", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 1.3397122910027974, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 11.94286273810605, "request_goodput": null, "output_throughput": 764.3432152387873, "total_token_throughput": "", "ttfts": [ 0.03514211699803127, 0.05226808499719482, 0.44446270300250035, 0.05195877300138818, 0.05190961900370894, 0.4486389930025325, 0.44819150200055446, 0.44818597600533394, 0.31471995700121624, 0.31485301799693843, 0.3146383870043792, 0.31465206499706255, 0.305763233001926, 0.3056597030008561, 0.30554367400327465, 0.30545302200334845 ], "max_output_tokens_per_s": 729.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 278.8775516887654, "median_ttft_ms": 310.2008100031526, "std_ttft_ms": 144.76929043386139, "p50_ttft_ms": 310.2008100031526, "p95_ttft_ms": 448.303374751049, "p99_ttft_ms": 448.5718693522358, "mean_tpot_ms": 5.011837853161856, "median_tpot_ms": 4.672280690433654, "std_tpot_ms": 0.6914666266669459, "p50_tpot_ms": 4.672280690433654, "p95_tpot_ms": 6.204654059541691, "p99_tpot_ms": 6.321669173825834, "mean_itl_ms": 5.026800468646856, "median_itl_ms": 4.63368200144032, "std_itl_ms": 2.0582603647032793, "p50_itl_ms": 4.63368200144032, "p95_itl_ms": 6.1037593957735226, "p99_itl_ms": 16.430620602041017, "mean_e2el_ms": 594.6233364379623, "median_e2el_ms": 600.3577219998988, "std_e2el_ms": 107.49530351306177, "p50_e2el_ms": 600.3577219998988, "p95_e2el_ms": 743.3701832505903, "p99_e2el_ms": 743.6364038501779, "requests_per_second": { "average": 11.94286273810605 }, "output_tokens_per_second": { "average": 764.3432152387873 }, "total_tokens_per_second": { "average": 3821.7160761939363 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 594.6233364379623, "median": 600.3577219998988, "p99": 743.6364038501779 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0002/attempts/001/repeats/002/longer-prompt/results.json", "repeat": 2, "elapsed_seconds": 7.622720455998206, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000003", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 1.4893044080017717, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 10.743270424793483, "request_goodput": null, "output_throughput": 687.5693071867829, "total_token_throughput": "", "ttfts": [ 0.045696023997152224, 0.023242057002789807, 0.04533636900305282, 0.04527706299995771, 0.5355662060028408, 0.5413501880029798, 0.5410136850041454, 0.5409236129999044, 0.3163320650055539, 0.31664596900373, 0.3165000279986998, 0.316503467001894, 0.3413702380057657, 0.34201090300484793, 0.34190259499882814, 0.3418892420013435 ], "max_output_tokens_per_s": 637.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 309.4724820020929, "median_ttft_ms": 329.00810350474785, "std_ttft_ms": 178.1003381331419, "p50_ttft_ms": 329.00810350474785, "p95_ttft_ms": 541.097810753854, "p99_ttft_ms": 541.2997125531547, "mean_tpot_ms": 5.605329295619976, "median_tpot_ms": 5.031576730130288, "std_tpot_ms": 1.2537969234386819, "p50_tpot_ms": 5.031576730130288, "p95_tpot_ms": 7.759407956352031, "p99_tpot_ms": 7.921016232545038, "mean_itl_ms": 5.622063643780922, "median_itl_ms": 4.844297996896785, "std_itl_ms": 2.8710066253392266, "p50_itl_ms": 4.844297996896785, "p95_itl_ms": 12.84701300173765, "p99_itl_ms": 17.974963560700417, "mean_e2el_ms": 662.6082276261513, "median_e2el_ms": 643.9937450013531, "std_e2el_ms": 109.27910270080841, "p50_e2el_ms": 643.9937450013531, "p95_e2el_ms": 834.1099285044038, "p99_e2el_ms": 834.2260465058644, "requests_per_second": { "average": 10.743270424793483 }, "output_tokens_per_second": { "average": 687.5693071867829 }, "total_tokens_per_second": { "average": 3437.8465359339143 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 662.6082276261513, "median": 643.9937450013531, "p99": 834.2260465058644 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0002/attempts/001/repeats/003/longer-prompt/results.json", "repeat": 3, "elapsed_seconds": 7.719939283997519, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000010", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 1.5390184670031886, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 10.3962365254496, "request_goodput": null, "output_throughput": 665.3591376287744, "total_token_throughput": "", "ttfts": [ 0.02977050199842779, 0.04582105900044553, 0.5105547880011727, 0.045409683996695094, 0.04533699600142427, 0.514721104002092, 0.5143610029990668, 0.5142772219987819, 0.3240128069955972, 0.3247512230009306, 0.32457838900154456, 0.3245338080014335, 0.34096484199835686, 0.3409456860026694, 0.34076978099619737, 0.3407294809949235 ], "max_output_tokens_per_s": 661.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 305.09614843685995, "median_ttft_ms": 332.74035199792706, "std_ttft_ms": 169.23719917245447, "p50_ttft_ms": 332.74035199792706, "p95_ttft_ms": 514.4510282498231, "p99_ttft_ms": 514.6670888516383, "mean_tpot_ms": 5.809010615076471, "median_tpot_ms": 5.533959769799988, "std_tpot_ms": 0.9960159851248245, "p50_tpot_ms": 5.533959769799988, "p95_tpot_ms": 7.338950261934466, "p99_tpot_ms": 7.441873519097239, "mean_itl_ms": 5.826353024855252, "median_itl_ms": 5.076936999103054, "std_itl_ms": 2.7844200420483816, "p50_itl_ms": 5.076936999103054, "p95_itl_ms": 11.91985899786232, "p99_itl_ms": 19.68018640036462, "mean_e2el_ms": 671.0638171866776, "median_e2el_ms": 681.40127299921, "std_e2el_ms": 114.47126682586835, "p50_e2el_ms": 681.40127299921, "p95_e2el_ms": 817.709320501308, "p99_e2el_ms": 817.9861401004018, "requests_per_second": { "average": 10.3962365254496 }, "output_tokens_per_second": { "average": 665.3591376287744 }, "total_tokens_per_second": { "average": 3326.795688143872 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 671.0638171866776, "median": 681.40127299921, "p99": 817.9861401004018 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0002/attempts/001/repeats/004/longer-prompt/results.json", "repeat": 4, "elapsed_seconds": 7.613627407998138, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000018", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 1.4713750970040564, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 10.874181595555365, "request_goodput": null, "output_throughput": 695.9476221155434, "total_token_throughput": "", "ttfts": [ 0.029147261004254688, 0.05108517700136872, 0.05075649799982784, 0.5290480769981514, 0.05064707100245869, 0.5339046239969321, 0.533538971998496, 0.5335230450000381, 0.3284411869972246, 0.3284230739955092, 0.32834842799638864, 0.3279019499968854, 0.3284286210036953, 0.3287044059979962, 0.3286867249989882, 0.32862133100570645 ], "max_output_tokens_per_s": 633.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 308.7004029371201, "median_ttft_ms": 328.43490400045994, "std_ttft_ms": 173.40863425784647, "p50_ttft_ms": 328.43490400045994, "p95_ttft_ms": 533.630384998105, "p99_ttft_ms": 533.8497761971666, "mean_tpot_ms": 5.532684105161486, "median_tpot_ms": 4.876745674650126, "std_tpot_ms": 1.188407086970128, "p50_tpot_ms": 4.876745674650126, "p95_tpot_ms": 7.591774285694036, "p99_tpot_ms": 7.7502333332674995, "mean_itl_ms": 5.549201835815642, "median_itl_ms": 4.7890939968056045, "std_itl_ms": 2.7436859609454762, "p50_itl_ms": 4.7890939968056045, "p95_itl_ms": 13.322027598042041, "p99_itl_ms": 15.93133292044513, "mean_e2el_ms": 657.2595015622937, "median_e2el_ms": 634.568738998496, "std_e2el_ms": 114.45568442066204, "p50_e2el_ms": 634.568738998496, "p95_e2el_ms": 840.6716622521344, "p99_e2el_ms": 840.9592356496432, "requests_per_second": { "average": 10.874181595555365 }, "output_tokens_per_second": { "average": 695.9476221155434 }, "total_tokens_per_second": { "average": 3479.7381105777167 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 657.2595015622937, "median": 634.568738998496, "p99": 840.9592356496432 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] } ], "execution": { "mode": "sequential" } }, { "trial_id": "trial-0004", "status": "failed", "failure": { "code": "server_exited_early", "message": "vLLM exited before becoming ready (code 1)\nLatest log output:\n(APIServer pid=3270) pydantic_core._pydantic_core.ValidationError: 1 validation error for SchedulerConfig\n(APIServer pid=3270) scheduler_config = SchedulerConfig(\n(APIServer pid=3270) ^^^^^^^^^^^^^^^^\n(APIServer pid=3270) File \"/opt/runtime/lib/python3.12/site-packages/pydantic/_internal/_dataclasses.py\", line 121, in __init__\n(APIServer pid=3270) s.__pydantic_validator__.validate_python(ArgsKwargs(args, kwargs), self_instance=s)\n(APIServer pid=3270) max_num_seqs\n(APIServer pid=3270) Input should be greater than or equal to 1 [type=greater_than_equal, input_value=0, input_type=int]\n(APIServer pid=3270) For further information visit https://errors.pydantic.dev/2.13/v/greater_than_equal", "retryable": false }, "benchmark_count": 0, "metrics": {}, "benchmarks": [], "execution": { "mode": "sequential" } }, { "trial_id": "trial-0005", "status": "completed", "failure": null, "benchmark_count": 8, "metrics": { "requests_per_second": { "average": 4.7196621263266225 }, "throughput_tokens_per_second": { "average": 202.29861121895337 }, "total_tokens_per_second": { "average": 811.9735263628658 }, "ttft_ms": { "average": 1277.0947474064656, "median": 1559.8339128760017, "p95": 1660.4323830617886, "p99": 1766.4607964107288 }, "tpot_ms": { "average": 4.773583232947496, "median": 4.453100659140757, "p95": 5.95316979775524, "p99": 8.422602152163396 }, "itl_ms": { "average": 4.881042413175497, "median": 4.484189187223819, "p99": 13.81216421863429 }, "end_to_end_ms": { "average": 1501.1102559612368, "median": 1770.244377499239, "p95": 1869.7713861565717, "p99": 1974.2653324323328 }, "total_time_seconds": { "average": 3.78007216662445 } }, "benchmarks": [ { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0005/attempts/001/repeats/001/short-chat/results.json", "repeat": 1, "elapsed_seconds": 8.659552694996819, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000223", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.546288053999888, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 6.283656703673443, "request_goodput": null, "output_throughput": 201.07701451755017, "total_token_throughput": "", "ttfts": [ 0.30908951200399315, 0.028104716002417263, 0.4566171830010717, 0.6075566750005237, 0.7504919549974147, 0.9132814269978553, 1.0624043559946585, 1.2078233749998617, 1.0563011609992827, 1.0546598520013504, 1.0455606690011336, 1.0486575129980338, 1.0269176789952326, 1.0259880329977022, 1.027785625999968, 1.0406638370041037 ], "max_output_tokens_per_s": 214.0, "max_concurrent_requests": 13, "rtfx": 0.0, "mean_ttft_ms": 853.8689730621627, "median_ttft_ms": 1027.3516524976003, "std_ttft_ms": 322.18313625432586, "p50_ttft_ms": 1027.3516524976003, "p95_ttft_ms": 1098.7591107459593, "p99_ttft_ms": 1186.010522149081, "mean_tpot_ms": 4.756267163341568, "median_tpot_ms": 4.4318362581121145, "std_tpot_ms": 1.0509401518466808, "p50_tpot_ms": 4.4318362581121145, "p95_tpot_ms": 6.070587185369673, "p99_tpot_ms": 8.183910649876774, "mean_itl_ms": 4.904598224551243, "median_itl_ms": 4.571648998535238, "std_itl_ms": 2.0675703437202455, "p50_itl_ms": 4.571648998535238, "p95_itl_ms": 6.185228994581848, "p99_itl_ms": 15.354795799066776, "mean_e2el_ms": 1001.3132551257513, "median_e2el_ms": 1170.0972649996402, "std_e2el_ms": 302.29995541671235, "p50_e2el_ms": 1170.0972649996402, "p95_e2el_ms": 1236.938343001384, "p99_e2el_ms": 1321.8584982030734, "requests_per_second": { "average": 6.283656703673443 }, "output_tokens_per_second": { "average": 201.07701451755017 }, "total_tokens_per_second": { "average": 603.2310435526505 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1001.3132551257513, "median": 1170.0972649996402, "p99": 1321.8584982030734 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0005/attempts/001/repeats/002/short-chat/results.json", "repeat": 2, "elapsed_seconds": 8.65808216699952, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000232", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.5879313699988415, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 6.182544168475056, "request_goodput": null, "output_throughput": 197.8414133912018, "total_token_throughput": "", "ttfts": [ 0.031632594000257086, 0.39893797299737344, 0.5550603209994733, 0.6977102460004971, 0.8410566249949625, 0.9815426749992184, 1.1300348930017208, 1.2763994889974128, 1.0328944690045319, 1.023786468002072, 1.0251737340004183, 1.0220197050002753, 1.0371429510050802, 1.0408235869981581, 1.0369904649996897, 1.0432048689981457 ], "max_output_tokens_per_s": 215.0, "max_concurrent_requests": 13, "rtfx": 0.0, "mean_ttft_ms": 885.9006914999554, "median_ttft_ms": 1024.4801010012452, "std_ttft_ms": 307.0036580971227, "p50_ttft_ms": 1024.4801010012452, "p95_ttft_ms": 1166.6260420006438, "p99_ttft_ms": 1254.444799598059, "mean_tpot_ms": 4.831370145174949, "median_tpot_ms": 4.384236790252955, "std_tpot_ms": 1.718599812736162, "p50_tpot_ms": 4.384236790252955, "p95_tpot_ms": 6.363708007953509, "p99_tpot_ms": 10.44714653066729, "mean_itl_ms": 4.961411960677998, "median_itl_ms": 4.47236900072312, "std_itl_ms": 2.539252677301056, "p50_itl_ms": 4.47236900072312, "p95_itl_ms": 6.62505309883272, "p99_itl_ms": 19.672022596205377, "mean_e2el_ms": 1035.6731660003788, "median_e2el_ms": 1163.1641359963396, "std_e2el_ms": 270.65376121385424, "p50_e2el_ms": 1163.1641359963396, "p95_e2el_ms": 1301.2676630023634, "p99_e2el_ms": 1386.2316326034488, "requests_per_second": { "average": 6.182544168475056 }, "output_tokens_per_second": { "average": 197.8414133912018 }, "total_tokens_per_second": { "average": 593.5242401736053 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1035.6731660003788, "median": 1163.1641359963396, "p99": 1386.2316326034488 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0005/attempts/001/repeats/003/short-chat/results.json", "repeat": 3, "elapsed_seconds": 8.777784996003902, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000241", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.5384760269953404, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 6.302994328033246, "request_goodput": null, "output_throughput": 201.69581849706387, "total_token_throughput": "", "ttfts": [ 0.3628433570047491, 0.03280247899965616, 0.509614041999157, 0.6609333550004521, 0.8074060359940631, 1.1025072890042793, 0.951652556002955, 1.2490482200009865, 1.044544449003297, 1.0417065459987498, 1.033856221998576, 1.0318258020051871, 1.0364326460039592, 1.030666033999296, 1.0219438190033543, 1.0157122339951457 ], "max_output_tokens_per_s": 213.0, "max_concurrent_requests": 13, "rtfx": 0.0, "mean_ttft_ms": 870.8434428758665, "median_ttft_ms": 1026.304926501325, "std_ttft_ms": 311.330124085314, "p50_ttft_ms": 1026.304926501325, "p95_ttft_ms": 1139.142521753456, "p99_ttft_ms": 1227.0670803514802, "mean_tpot_ms": 4.729644600751446, "median_tpot_ms": 4.35971193550241, "std_tpot_ms": 1.450124568778073, "p50_tpot_ms": 4.35971193550241, "p95_tpot_ms": 5.991270088587395, "p99_tpot_ms": 9.467309604799317, "mean_itl_ms": 4.867027265530421, "median_itl_ms": 4.413136997754918, "std_itl_ms": 2.3847430528085054, "p50_itl_ms": 4.413136997754918, "p95_itl_ms": 6.155423398013226, "p99_itl_ms": 15.422780145963769, "mean_e2el_ms": 1017.4624254991613, "median_e2el_ms": 1158.1969894978101, "std_e2el_ms": 281.2300606222928, "p50_e2el_ms": 1158.1969894978101, "p95_e2el_ms": 1275.7724399998551, "p99_e2el_ms": 1364.2307256028287, "requests_per_second": { "average": 6.302994328033246 }, "output_tokens_per_second": { "average": 201.69581849706387 }, "total_tokens_per_second": { "average": 605.0874554911917 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1017.4624254991613, "median": 1158.1969894978101, "p99": 1364.2307256028287 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0005/attempts/001/repeats/004/short-chat/results.json", "repeat": 4, "elapsed_seconds": 8.6339561509958, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000250", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 2.592879363000975, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 6.170746016305882, "request_goodput": null, "output_throughput": 197.46387252178823, "total_token_throughput": "", "ttfts": [ 0.01981718400202226, 0.3662477370016859, 0.9677559190022293, 0.5110977150034159, 0.6726982809996116, 0.8201854479993926, 1.1180528019976919, 1.2690984189976007, 1.0618476489980821, 1.0630690540056094, 1.0551762920003966, 1.0547913789996528, 1.0612696140015032, 1.0607980079948902, 1.0546112629963318, 1.055240073997993 ], "max_output_tokens_per_s": 207.0, "max_concurrent_requests": 13, "rtfx": 0.0, "mean_ttft_ms": 888.2348023748818, "median_ttft_ms": 1054.9838355000247, "std_ttft_ms": 321.68946057024704, "p50_ttft_ms": 1054.9838355000247, "p95_ttft_ms": 1155.814206247669, "p99_ttft_ms": 1246.4415764476141, "mean_tpot_ms": 4.862776870962285, "median_tpot_ms": 4.424360709714576, "std_tpot_ms": 1.541365505241078, "p50_tpot_ms": 4.424360709714576, "p95_tpot_ms": 6.3379880644905855, "p99_tpot_ms": 9.91236736767713, "mean_itl_ms": 5.014428752580375, "median_itl_ms": 4.497128997172695, "std_itl_ms": 2.3331662928608043, "p50_itl_ms": 4.497128997172695, "p95_itl_ms": 7.511265997891314, "p99_itl_ms": 17.214808397693552, "mean_e2el_ms": 1038.9808853747127, "median_e2el_ms": 1191.1624314998335, "std_e2el_ms": 289.8712305623962, "p50_e2el_ms": 1191.1624314998335, "p95_e2el_ms": 1294.8269342487038, "p99_e2el_ms": 1381.6254980491067, "requests_per_second": { "average": 6.170746016305882 }, "output_tokens_per_second": { "average": 197.46387252178823 }, "total_tokens_per_second": { "average": 592.3916175653646 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1038.9808853747127, "median": 1191.1624314998335, "p99": 1381.6254980491067 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0005/attempts/001/repeats/001/longer-prompt/results.json", "repeat": 1, "elapsed_seconds": 11.021894154997426, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000311", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 4.911961253004847, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.2573546849971886, "request_goodput": null, "output_throughput": 208.47069983982007, "total_token_throughput": "", "ttfts": [ 0.035003194003365934, 0.5365586330008227, 0.8226644650058006, 1.1092898310016608, 1.7075205989967799, 1.4114184720019693, 1.999136405000172, 2.305844625996542, 2.072057846999087, 2.0801919100049417, 2.080293705999793, 2.067560871997557, 2.055220294998435, 2.0642112190034823, 2.052382340996701, 2.047419226000784 ], "max_output_tokens_per_s": 217.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 1652.9233525629934, "median_ttft_ms": 2049.9007834987424, "std_ttft_ms": 652.6520435414258, "p50_ttft_ms": 2049.9007834987424, "p95_ttft_ms": 2136.6814359989803, "p99_ttft_ms": 2272.0119879970293, "mean_tpot_ms": 4.679218993050964, "median_tpot_ms": 4.4487231587857154, "std_tpot_ms": 0.8072444711506201, "p50_tpot_ms": 4.4487231587857154, "p95_tpot_ms": 5.473720888894316, "p99_tpot_ms": 7.318963669835254, "mean_itl_ms": 4.740357259303055, "median_itl_ms": 4.424468999786768, "std_itl_ms": 1.5784378249928512, "p50_itl_ms": 4.424468999786768, "p95_itl_ms": 6.107096801861189, "p99_itl_ms": 13.622791579546165, "mean_e2el_ms": 1947.7141491252041, "median_e2el_ms": 2326.128444998176, "std_e2el_ms": 622.1328736205782, "p50_e2el_ms": 2326.128444998176, "p95_e2el_ms": 2415.0680229977297, "p99_e2el_ms": 2551.074738999159, "requests_per_second": { "average": 3.2573546849971886 }, "output_tokens_per_second": { "average": 208.47069983982007 }, "total_tokens_per_second": { "average": 1042.3534991991003 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1947.7141491252041, "median": 2326.128444998176, "p99": 2551.074738999159 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0005/attempts/001/repeats/002/longer-prompt/results.json", "repeat": 2, "elapsed_seconds": 11.234453041004599, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000323", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 5.046052281999437, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.170795526054319, "request_goodput": null, "output_throughput": 202.9309136674764, "total_token_throughput": "", "ttfts": [ 0.03775041000335477, 0.4674380510041374, 1.0482043899974087, 0.746875603996159, 1.3364528860038263, 1.6487193049979396, 2.3173639079977875, 1.9963047410055879, 2.1537109230048372, 2.1638208119984483, 2.1581528700044146, 2.2423221550052403, 2.227632537003956, 2.188575360996765, 2.1617962199961767, 2.15840053899592 ], "max_output_tokens_per_s": 211.0, "max_concurrent_requests": 12, "rtfx": 0.0, "mean_ttft_ms": 1690.8450445007475, "median_ttft_ms": 2155.931896504626, "std_ttft_ms": 710.7850275275631, "p50_ttft_ms": 2155.931896504626, "p95_ttft_ms": 2261.082593253377, "p99_ttft_ms": 2306.1076450489054, "mean_tpot_ms": 4.805702897816186, "median_tpot_ms": 4.562300785769439, "std_tpot_ms": 0.5974491073352941, "p50_tpot_ms": 4.562300785769439, "p95_tpot_ms": 5.963231968226388, "p99_tpot_ms": 6.521628882529469, "mean_itl_ms": 4.883217283276603, "median_itl_ms": 4.553726001176983, "std_itl_ms": 1.4420041379203516, "p50_itl_ms": 4.553726001176983, "p95_itl_ms": 6.66760959829844, "p99_itl_ms": 10.83313640934648, "mean_e2el_ms": 1993.6043270631671, "median_e2el_ms": 2436.772752498655, "std_e2el_ms": 696.9930912731097, "p50_e2el_ms": 2436.772752498655, "p95_e2el_ms": 2546.679506500368, "p99_e2el_ms": 2588.7163069011876, "requests_per_second": { "average": 3.170795526054319 }, "output_tokens_per_second": { "average": 202.9309136674764 }, "total_tokens_per_second": { "average": 1014.654568337382 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1993.6043270631671, "median": 2436.772752498655, "p99": 2588.7163069011876 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0005/attempts/001/repeats/003/longer-prompt/results.json", "repeat": 3, "elapsed_seconds": 11.467014321002353, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000334", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 4.993134445001488, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.2043999968831667, "request_goodput": null, "output_throughput": 205.08159980052267, "total_token_throughput": "", "ttfts": [ 0.024475443999108393, 0.538853014993947, 1.150803005999478, 0.8347773300047265, 1.4535197360019083, 2.0406046909993165, 1.7520350300037535, 2.3338460409941035, 2.1015985289996024, 2.091029075003462, 2.064455733001523, 2.0853572260020883, 2.093083571002353, 2.088744050000969, 2.100827139998728, 2.094860051998694 ], "max_output_tokens_per_s": 214.0, "max_concurrent_requests": 11, "rtfx": 0.0, "mean_ttft_ms": 1678.054354312735, "median_ttft_ms": 2074.9064795018057, "std_ttft_ms": 661.3020293180007, "p50_ttft_ms": 2074.9064795018057, "p95_ttft_ms": 2159.6604069982277, "p99_ttft_ms": 2299.0089141949284, "mean_tpot_ms": 4.76821464980319, "median_tpot_ms": 4.515632825449945, "std_tpot_ms": 0.8435193780867672, "p50_tpot_ms": 4.515632825449945, "p95_tpot_ms": 5.698924480153466, "p99_tpot_ms": 7.520586280149891, "mean_itl_ms": 4.845123986907281, "median_itl_ms": 4.5001420003245585, "std_itl_ms": 1.5341501053119193, "p50_itl_ms": 4.5001420003245585, "p95_itl_ms": 6.809908148716203, "p99_itl_ms": 11.118831189596715, "mean_e2el_ms": 1978.451877250336, "median_e2el_ms": 2368.7572250019, "std_e2el_ms": 626.540071838852, "p50_e2el_ms": 2368.7572250019, "p95_e2el_ms": 2441.5552745012974, "p99_e2el_ms": 2581.7758084966044, "requests_per_second": { "average": 3.2043999968831667 }, "output_tokens_per_second": { "average": 205.08159980052267 }, "total_tokens_per_second": { "average": 1025.4079990026135 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1978.451877250336, "median": 2368.7572250019, "p99": 2581.7758084966044 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0005/attempts/001/repeats/004/longer-prompt/results.json", "repeat": 4, "elapsed_seconds": 11.529255111003295, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000346", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 5.023854538994783, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 3.184805586190682, "request_goodput": null, "output_throughput": 203.82755751620365, "total_token_throughput": "", "ttfts": [ 0.06849664200126426, 0.6207918969957973, 1.2017954830007511, 0.9096695769985672, 1.4973601599995163, 1.7968431869958295, 2.092817289994855, 2.384319119999418, 2.0644566870032577, 2.064029080000182, 2.0830266940029105, 2.071654583000054, 2.0750047430046834, 2.0681929839993245, 2.065166569002031, 2.073772392999672 ], "max_output_tokens_per_s": 216.0, "max_concurrent_requests": 11, "rtfx": 0.0, "mean_ttft_ms": 1696.087318062382, "median_ttft_ms": 2064.8116280026443, "std_ttft_ms": 635.9809047590047, "p50_ttft_ms": 2064.8116280026443, "p95_ttft_ms": 2165.6927474959957, "p99_ttft_ms": 2340.5938454987336, "mean_tpot_ms": 4.7554705426793795, "median_tpot_ms": 4.498002809538906, "std_tpot_ms": 0.9925711415184153, "p50_tpot_ms": 4.498002809538906, "p95_tpot_ms": 5.725927698366585, "p99_tpot_ms": 8.008904231772044, "mean_itl_ms": 4.832174572577001, "median_itl_ms": 4.440892502316274, "std_itl_ms": 3.5683597457874288, "p50_itl_ms": 4.440892502316274, "p95_itl_ms": 5.796214797737775, "p99_itl_ms": 7.258147631655489, "mean_e2el_ms": 1995.681962251183, "median_e2el_ms": 2347.675775501557, "std_e2el_ms": 596.9079630258135, "p50_e2el_ms": 2347.675775501557, "p95_e2el_ms": 2446.062905000872, "p99_e2el_ms": 2618.609450603253, "requests_per_second": { "average": 3.184805586190682 }, "output_tokens_per_second": { "average": 203.82755751620365 }, "total_tokens_per_second": { "average": 1019.1377875810183 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 1995.681962251183, "median": 2347.675775501557, "p99": 2618.609450603253 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] } ], "execution": { "mode": "sequential" } }, { "trial_id": "trial-0006", "status": "completed", "failure": null, "benchmark_count": 8, "metrics": { "requests_per_second": { "average": 15.791673588211626 }, "throughput_tokens_per_second": { "average": 682.9141973552589 }, "total_tokens_per_second": { "average": 2759.0651621957245 }, "ttft_ms": { "average": 238.7509456174257, "median": 249.21040399976846, "p95": 414.4180965308806, "p99": 414.5802775071388 }, "tpot_ms": { "average": 5.5916073990924495, "median": 4.928614202542261, "p95": 7.784893071102832, "p99": 7.9777726120423 }, "itl_ms": { "average": 5.617192695456992, "median": 4.8506401226404705, "p99": 20.081820415216487 }, "end_to_end_ms": { "average": 499.2461856956538, "median": 478.72191068881875, "p95": 642.3504993431379, "p99": 642.5053772697538 }, "total_time_seconds": { "average": 1.1164273932508877 } }, "benchmarks": [ { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0006/attempts/001/repeats/001/short-chat/results.json", "repeat": 1, "elapsed_seconds": 7.1546463900012895, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000416", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8112196479996783, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 19.723388159363648, "request_goodput": null, "output_throughput": 631.1484210996367, "total_token_throughput": "", "ttfts": [ 0.03841011600161437, 0.0682342840009369, 0.06782932199712377, 0.06778573500196217, 0.33783163099724334, 0.3428719869989436, 0.3425434689997928, 0.3424299019970931, 0.17284762800409226, 0.1719682820039452, 0.17181246999825817, 0.1717898850038182, 0.17020101700472878, 0.17101486399769783, 0.17090972600271925, 0.17087564800021937 ], "max_output_tokens_per_s": 509.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 186.20974787563682, "median_ttft_ms": 171.40237450075801, "std_ttft_ms": 100.6017220712373, "p50_ttft_ms": 171.40237450075801, "p95_ttft_ms": 342.6255984995805, "p99_ttft_ms": 342.822709299071, "mean_tpot_ms": 5.811729042350987, "median_tpot_ms": 4.906782387068586, "std_tpot_ms": 1.706983683767654, "p50_tpot_ms": 4.906782387068586, "p95_tpot_ms": 8.750143524124606, "p99_tpot_ms": 9.21754315651572, "mean_itl_ms": 5.847097943218979, "median_itl_ms": 4.866797993599903, "std_itl_ms": 3.5648085843209465, "p50_itl_ms": 4.866797993599903, "p95_itl_ms": 13.508411795191842, "p99_itl_ms": 22.79145808162865, "mean_e2el_ms": 366.3733481885174, "median_e2el_ms": 325.97123900268343, "std_e2el_ms": 73.88987028890638, "p50_e2el_ms": 325.97123900268343, "p95_e2el_ms": 495.0373892497737, "p99_e2el_ms": 495.2473898534663, "requests_per_second": { "average": 19.723388159363648 }, "output_tokens_per_second": { "average": 631.1484210996367 }, "total_tokens_per_second": { "average": 1893.44526329891 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 366.3733481885174, "median": 325.97123900268343, "p99": 495.2473898534663 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0006/attempts/001/repeats/002/short-chat/results.json", "repeat": 2, "elapsed_seconds": 6.95577813199634, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000423", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8136358179981471, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 19.66481765683088, "request_goodput": null, "output_throughput": 629.2741650185882, "total_token_throughput": "", "ttfts": [ 0.03677871599938953, 0.020199465005134698, 0.036272806995839346, 0.03624144900095416, 0.3528462380054407, 0.3475567410059739, 0.35235094300151104, 0.35194280299765524, 0.16798426199966343, 0.16753854200214846, 0.16745056299987482, 0.1669244609947782, 0.17129342700354755, 0.17106858800252667, 0.17094367899699137, 0.17083610699774 ], "max_output_tokens_per_s": 509.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 180.51429943807307, "median_ttft_ms": 169.4101844987017, "std_ttft_ms": 113.34082809592321, "p50_ttft_ms": 169.4101844987017, "p95_ttft_ms": 352.47476675249345, "p99_ttft_ms": 352.77194375485124, "mean_tpot_ms": 6.065791737925, "median_tpot_ms": 4.8159044193697245, "std_tpot_ms": 2.2594587602439087, "p50_tpot_ms": 4.8159044193697245, "p95_tpot_ms": 9.972126282234733, "p99_tpot_ms": 10.190962133879845, "mean_itl_ms": 6.102706785039866, "median_itl_ms": 4.756026995892171, "std_itl_ms": 4.159124601955662, "p50_itl_ms": 4.756026995892171, "p95_itl_ms": 16.14808579761302, "p99_itl_ms": 26.072957319847774, "mean_e2el_ms": 368.5538433137481, "median_e2el_ms": 329.46754650402, "std_e2el_ms": 75.84024588294334, "p50_e2el_ms": 329.46754650402, "p95_e2el_ms": 500.0759187514632, "p99_e2el_ms": 500.2367949546169, "requests_per_second": { "average": 19.66481765683088 }, "output_tokens_per_second": { "average": 629.2741650185882 }, "total_tokens_per_second": { "average": 1887.8224950557646 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 368.5538433137481, "median": 329.46754650402, "p99": 500.2367949546169 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0006/attempts/001/repeats/003/short-chat/results.json", "repeat": 3, "elapsed_seconds": 6.812973029002023, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000430", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.6723051150038373, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 23.798718235110673, "request_goodput": null, "output_throughput": 761.5589835235415, "total_token_throughput": "", "ttfts": [ 0.05109349000122165, 0.05937528800131986, 0.05896797800232889, 0.058866316998319235, 0.2155222040018998, 0.22039907099679112, 0.2203767870014417, 0.2203573079968919, 0.16006050300347852, 0.1599320320019615, 0.1598398089990951, 0.15940452399809146, 0.16410676400118973, 0.16308084999764105, 0.1628605130026699, 0.16286370900343172 ], "max_output_tokens_per_s": 509.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 149.81919668798582, "median_ttft_ms": 161.4605080030742, "std_ttft_ms": 58.53702075936226, "p50_ttft_ms": 161.4605080030742, "p95_ttft_ms": 220.38235800027905, "p99_ttft_ms": 220.3957283974887, "mean_tpot_ms": 4.735853592753683, "median_tpot_ms": 4.693063596806173, "std_tpot_ms": 0.1454767960250373, "p50_tpot_ms": 4.693063596806173, "p95_tpot_ms": 4.956408540296721, "p99_tpot_ms": 5.025465733952813, "mean_itl_ms": 4.7646757910919755, "median_itl_ms": 4.684849001932889, "std_itl_ms": 0.6028239863420994, "p50_itl_ms": 4.684849001932889, "p95_itl_ms": 5.749300602474249, "p99_itl_ms": 6.977919760684012, "mean_e2el_ms": 296.63065806335, "median_e2el_ms": 306.86098350633984, "std_e2el_ms": 54.2694371563134, "p50_e2el_ms": 306.86098350633984, "p95_e2el_ms": 362.4280829990312, "p99_e2el_ms": 362.4660845965991, "requests_per_second": { "average": 23.798718235110673 }, "output_tokens_per_second": { "average": 761.5589835235415 }, "total_tokens_per_second": { "average": 2284.6769505706247 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 296.63065806335, "median": 306.86098350633984, "p99": 362.4660845965991 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0006/attempts/001/repeats/004/short-chat/results.json", "repeat": 4, "elapsed_seconds": 7.0664472810021834, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000437", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8532739900037996, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 18.75130402126608, "request_goodput": null, "output_throughput": 600.0417286805146, "total_token_throughput": "", "ttfts": [ 0.019202572999347467, 0.040380898004514165, 0.03994035600044299, 0.03994003700063331, 0.3150797890048125, 0.31935332900320645, 0.31930909699440235, 0.3192038950001006, 0.18871391999709886, 0.18946463400061475, 0.18934160900244024, 0.18931920400063973, 0.20211530500091612, 0.19964436500595184, 0.19941137800196884, 0.19946379400062142 ], "max_output_tokens_per_s": 509.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 185.61776143860698, "median_ttft_ms": 194.4380060012918, "std_ttft_ms": 100.77783658961381, "p50_ttft_ms": 194.4380060012918, "p95_ttft_ms": 319.3201549966034, "p99_ttft_ms": 319.34669420188584, "mean_tpot_ms": 6.330632143119381, "median_tpot_ms": 5.603938677414853, "std_tpot_ms": 1.468398161252718, "p50_tpot_ms": 5.603938677414853, "p95_tpot_ms": 8.856978999897219, "p99_tpot_ms": 9.152816935485106, "mean_itl_ms": 6.369159659246217, "median_itl_ms": 5.4748510010540485, "std_itl_ms": 3.0565858437660105, "p50_itl_ms": 5.4748510010540485, "p95_itl_ms": 14.206379403185531, "p99_itl_ms": 17.37412411865073, "mean_e2el_ms": 381.8673578753078, "median_e2el_ms": 365.7905359978031, "std_e2el_ms": 63.9783559552863, "p50_e2el_ms": 365.7905359978031, "p95_e2el_ms": 486.8894214996544, "p99_e2el_ms": 486.9139531037945, "requests_per_second": { "average": 18.75130402126608 }, "output_tokens_per_second": { "average": 600.0417286805146 }, "total_tokens_per_second": { "average": 1800.1251860415437 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 381.8673578753078, "median": 365.7905359978031, "p99": 486.9139531037945 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0006/attempts/001/repeats/001/longer-prompt/results.json", "repeat": 1, "elapsed_seconds": 7.844363809999777, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000452", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 1.572918085999845, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 10.172176251523867, "request_goodput": null, "output_throughput": 651.0192800975275, "total_token_throughput": "", "ttfts": [ 0.633974948003015, 0.04124744400178315, 0.0758821430063108, 0.07581834500160767, 0.07571236100193346, 0.6282826180031407, 0.6327881559991511, 0.6323587420047261, 0.33088244700047653, 0.33406402399850776, 0.33394384499843, 0.33391659100016113, 0.3317315349995624, 0.3338283070042962, 0.33370231300068554, 0.3336765599960927 ], "max_output_tokens_per_s": 553.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 341.3631486887425, "median_ttft_ms": 333.7653100024909, "std_ttft_ms": 199.9572352740344, "p50_ttft_ms": 333.7653100024909, "p95_ttft_ms": 633.084854000117, "p99_ttft_ms": 633.7969292024354, "mean_tpot_ms": 5.840918505934193, "median_tpot_ms": 4.924174849177704, "std_tpot_ms": 1.7171715361539852, "p50_tpot_ms": 4.924174849177704, "p95_tpot_ms": 8.805510972259013, "p99_tpot_ms": 9.090898867442673, "mean_itl_ms": 5.858356186054967, "median_itl_ms": 4.836086998693645, "std_itl_ms": 4.729134581202211, "p50_itl_ms": 4.836086998693645, "p95_itl_ms": 7.748088396328965, "p99_itl_ms": 32.57491056487197, "mean_e2el_ms": 709.3410145625967, "median_e2el_ms": 638.6236934995395, "std_e2el_ms": 134.74561668715896, "p50_e2el_ms": 638.6236934995395, "p95_e2el_ms": 943.7119249978423, "p99_e2el_ms": 944.4304946035118, "requests_per_second": { "average": 10.172176251523867 }, "output_tokens_per_second": { "average": 651.0192800975275 }, "total_tokens_per_second": { "average": 3255.0964004876378 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 709.3410145625967, "median": 638.6236934995395, "p99": 944.4304946035118 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0006/attempts/001/repeats/002/longer-prompt/results.json", "repeat": 2, "elapsed_seconds": 7.493998215002648, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000459", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 1.3879283399946871, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 11.527972690622663, "request_goodput": null, "output_throughput": 737.7902521998504, "total_token_throughput": "", "ttfts": [ 0.0238137589985854, 0.03516501600097399, 0.03473907699662959, 0.034763554998789914, 0.4658234430025914, 0.4701449050044175, 0.4701296450002701, 0.47010772899375297, 0.3286293920027674, 0.32891411400487414, 0.32892251500015846, 0.32843445999606047, 0.3253155389975291, 0.3229173939980683, 0.32271425900398754, 0.3227324420004152 ], "max_output_tokens_per_s": 689.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 288.32920274999196, "median_ttft_ms": 326.87499949679477, "std_ttft_ms": 159.05536364938047, "p50_ttft_ms": 326.87499949679477, "p95_ttft_ms": 470.13346000130696, "p99_ttft_ms": 470.1426160037954, "mean_tpot_ms": 5.260575136891555, "median_tpot_ms": 4.850490095180696, "std_tpot_ms": 0.894306577972423, "p50_tpot_ms": 4.850490095180696, "p95_tpot_ms": 6.796524404813277, "p99_tpot_ms": 6.856293719061392, "mean_itl_ms": 5.276280109440753, "median_itl_ms": 4.7301499944296665, "std_itl_ms": 2.641669317540029, "p50_itl_ms": 4.7301499944296665, "p95_itl_ms": 6.880160805303603, "p99_itl_ms": 20.37462635547854, "mean_e2el_ms": 619.7454363741599, "median_e2el_ms": 622.9409599982318, "std_e2el_ms": 112.48152030431471, "p50_e2el_ms": 622.9409599982318, "p95_e2el_ms": 778.6895472472679, "p99_e2el_ms": 778.6921662471286, "requests_per_second": { "average": 11.527972690622663 }, "output_tokens_per_second": { "average": 737.7902521998504 }, "total_tokens_per_second": { "average": 3688.951260999252 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 619.7454363741599, "median": 622.9409599982318, "p99": 778.6921662471286 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0006/attempts/001/repeats/003/longer-prompt/results.json", "repeat": 3, "elapsed_seconds": 7.578758106996247, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000507", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 1.4003060090035433, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 11.42607394178476, "request_goodput": null, "output_throughput": 731.2687322742246, "total_token_throughput": "", "ttfts": [ 0.01850648500112584, 0.034793234000972006, 0.034277088998351246, 0.034278578001249116, 0.47948567399726016, 0.48415917200327385, 0.48411171900079353, 0.48410667099960847, 0.31725855200056685, 0.3166616320013418, 0.3166441279972787, 0.3165767279933789, 0.3109454259974882, 0.3107344859963632, 0.3106656089948956, 0.3105979190004291 ], "max_output_tokens_per_s": 697.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 285.23769387402353, "median_ttft_ms": 313.76107699543354, "std_ttft_ms": 162.56026587758782, "p50_ttft_ms": 313.76107699543354, "p95_ttft_ms": 484.1235822514136, "p99_ttft_ms": 484.1520540529018, "mean_tpot_ms": 5.3161824434648715, "median_tpot_ms": 4.795769746066533, "std_tpot_ms": 0.995741542606387, "p50_tpot_ms": 4.795769746066533, "p95_tpot_ms": 7.032349662675563, "p99_tpot_ms": 7.131317576995426, "mean_itl_ms": 5.332053241792566, "median_itl_ms": 4.74148899957072, "std_itl_ms": 2.096741612665719, "p50_itl_ms": 4.74148899957072, "p95_itl_ms": 9.111145400675014, "p99_itl_ms": 14.237252997409087, "mean_e2el_ms": 620.1571878123104, "median_e2el_ms": 613.3668149996083, "std_e2el_ms": 108.6887432820569, "p50_e2el_ms": 613.3668149996083, "p95_e2el_ms": 781.1800544986909, "p99_e2el_ms": 781.247512500704, "requests_per_second": { "average": 11.42607394178476 }, "output_tokens_per_second": { "average": 731.2687322742246 }, "total_tokens_per_second": { "average": 3656.343661371123 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 620.1571878123104, "median": 613.3668149996083, "p99": 781.247512500704 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0006/attempts/001/repeats/004/longer-prompt/results.json", "repeat": 4, "elapsed_seconds": 7.446704745001625, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000514", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 1.4198321400035638, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 11.268937749190435, "request_goodput": null, "output_throughput": 721.2120159481879, "total_token_throughput": "", "ttfts": [ 0.026143954004510306, 0.037669407996872906, 0.03734474399971077, 0.03720395200070925, 0.48833822799497284, 0.49321693199453875, 0.49319435299548786, 0.49318555100035155, 0.31935741400229745, 0.31901020200166386, 0.318781490001129, 0.31880326499958755, 0.32578413099690806, 0.32632707199809374, 0.3261608749962761, 0.3261426559984102 ], "max_output_tokens_per_s": 677.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 292.916514186345, "median_ttft_ms": 322.57077249960275, "std_ttft_ms": 164.44569086182622, "p50_ttft_ms": 322.57077249960275, "p95_ttft_ms": 493.1999977452506, "p99_ttft_ms": 493.2135451446811, "mean_tpot_ms": 5.371176590299926, "median_tpot_ms": 4.838789849253814, "std_tpot_ms": 1.004331858987018, "p50_tpot_ms": 4.838789849253814, "p95_tpot_ms": 7.109102182521528, "p99_tpot_ms": 7.1568827730054245, "mean_itl_ms": 5.387211847770612, "median_itl_ms": 4.714869995950721, "std_itl_ms": 2.615164640196523, "p50_itl_ms": 4.714869995950721, "p95_itl_ms": 10.361764604749624, "p99_itl_ms": 20.251314123161137, "mean_e2el_ms": 631.3006393752403, "median_e2el_ms": 626.753512002324, "std_e2el_ms": 108.71452618669623, "p50_e2el_ms": 626.753512002324, "p95_e2el_ms": 790.7916555013799, "p99_e2el_ms": 790.8086222982092, "requests_per_second": { "average": 11.268937749190435 }, "output_tokens_per_second": { "average": 721.2120159481879 }, "total_tokens_per_second": { "average": 3606.0600797409393 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 631.3006393752403, "median": 626.753512002324, "p99": 790.8086222982092 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] } ], "execution": { "mode": "sequential" } }, { "trial_id": "trial-0008", "status": "failed", "failure": { "code": "server_exited_early", "message": "vLLM exited before becoming ready (code 1)\nLatest log output:\n(APIServer pid=5418) pydantic_core._pydantic_core.ValidationError: 1 validation error for SchedulerConfig\n(APIServer pid=5418) scheduler_config = SchedulerConfig(\n(APIServer pid=5418) ^^^^^^^^^^^^^^^^\n(APIServer pid=5418) File \"/opt/runtime/lib/python3.12/site-packages/pydantic/_internal/_dataclasses.py\", line 121, in __init__\n(APIServer pid=5418) s.__pydantic_validator__.validate_python(ArgsKwargs(args, kwargs), self_instance=s)\n(APIServer pid=5418) max_num_seqs\n(APIServer pid=5418) Input should be greater than or equal to 1 [type=greater_than_equal, input_value=0, input_type=int]\n(APIServer pid=5418) For further information visit https://errors.pydantic.dev/2.13/v/greater_than_equal", "retryable": false }, "benchmark_count": 0, "metrics": {}, "benchmarks": [], "execution": { "mode": "sequential" } }, { "trial_id": "trial-0007", "status": "completed", "failure": null, "benchmark_count": 8, "metrics": { "requests_per_second": { "average": 24.216098868319218 }, "throughput_tokens_per_second": { "average": 1081.0585160757298 }, "total_tokens_per_second": { "average": 4467.748957385249 }, "ttft_ms": { "average": 88.47178687500445, "median": 82.72852750087623, "p95": 156.42199246804012, "p99": 156.6178545942421 }, "tpot_ms": { "average": 5.435957775425205, "median": 5.464326963232018, "p95": 6.168003345017671, "p99": 6.35652624914674 }, "itl_ms": { "average": 5.551941527340949, "median": 4.823274874979688, "p99": 14.001305282563408 }, "end_to_end_ms": { "average": 346.62613297666667, "median": 346.58911731230546, "p95": 446.5850142198633, "p99": 446.70053994436785 }, "total_time_seconds": { "average": 0.6947118283751479 } }, "benchmarks": [ { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0007/validation-001/attempts/001/repeats/001/short-chat/results.json", "repeat": 1, "elapsed_seconds": 6.744133928994415, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000717", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.6112767329977942, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 26.174724369981437, "request_goodput": null, "output_throughput": 837.591179839406, "total_token_throughput": "", "ttfts": [ 0.2778750840006978, 0.2835179169996991, 0.28311221500189276, 0.28301535599894123, 0.2830442109989235, 0.2823239389981609, 0.2823624359953101, 0.2823782690029475, 0.018397541003650986, 0.024571382004069164, 0.023978101002285257, 0.02393151100113755, 0.023608032999618445, 0.023581137000292074, 0.023258681998413522, 0.023237252004037146 ], "max_output_tokens_per_s": 498.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 152.6370666256298, "median_ttft_ms": 151.2232330023835, "std_ttft_ms": 129.5784275351294, "p50_ttft_ms": 151.2232330023835, "p95_ttft_ms": 283.21364050134434, "p99_ttft_ms": 283.45706170002813, "mean_tpot_ms": 4.913840627024558, "median_tpot_ms": 4.963285322563963, "std_tpot_ms": 0.11045009832334299, "p50_tpot_ms": 4.963285322563963, "p95_tpot_ms": 5.041501258024401, "p99_tpot_ms": 5.145395941919208, "mean_itl_ms": 5.0565707220299085, "median_itl_ms": 4.857616499066353, "std_itl_ms": 0.8251573939330935, "p50_itl_ms": 4.857616499066353, "p95_itl_ms": 6.878449398573139, "p99_itl_ms": 7.4146885633672355, "mean_e2el_ms": 304.96612606339113, "median_e2el_ms": 304.7725284995977, "std_e2el_ms": 132.33677977564562, "p50_e2el_ms": 304.7725284995977, "p95_e2el_ms": 438.097848002144, "p99_e2el_ms": 438.1696032007312, "requests_per_second": { "average": 26.174724369981437 }, "output_tokens_per_second": { "average": 837.591179839406 }, "total_tokens_per_second": { "average": 2512.773539518218 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 304.96612606339113, "median": 304.7725284995977, "p99": 438.1696032007312 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0007/validation-001/attempts/001/repeats/002/short-chat/results.json", "repeat": 2, "elapsed_seconds": 6.632852417998947, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000724", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.5858311470001354, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 27.31162397549392, "request_goodput": null, "output_throughput": 873.9719672158054, "total_token_throughput": "", "ttfts": [ 0.2503068309961236, 0.25583033799921395, 0.2554858779985807, 0.2554024409982958, 0.2547558080041199, 0.2548297469984391, 0.2547734429972479, 0.2542414159979671, 0.018787512999551836, 0.02550678599800449, 0.02535829399857903, 0.025321038003312424, 0.024740801003645174, 0.02467912899737712, 0.02463556400471134, 0.024256673001218587 ], "max_output_tokens_per_s": 498.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 139.30698124977425, "median_ttft_ms": 137.90680849706405, "std_ttft_ms": 115.16138323033395, "p50_ttft_ms": 137.90680849706405, "p95_ttft_ms": 255.571992998739, "p99_ttft_ms": 255.77866899911893, "mean_tpot_ms": 4.928647651233346, "median_tpot_ms": 4.983112370903273, "std_tpot_ms": 0.10580835691798218, "p50_tpot_ms": 4.983112370903273, "p95_tpot_ms": 5.075063056386669, "p99_tpot_ms": 5.148954624230335, "mean_itl_ms": 5.071806744802178, "median_itl_ms": 4.923996501020156, "std_itl_ms": 0.8835485319621355, "p50_itl_ms": 4.923996501020156, "p95_itl_ms": 6.720698545177584, "p99_itl_ms": 7.856087140753516, "mean_e2el_ms": 292.095058438008, "median_e2el_ms": 291.98982599700685, "std_e2el_ms": 117.6748273834149, "p50_e2el_ms": 291.98982599700685, "p95_e2el_ms": 410.45431524980813, "p99_e2el_ms": 410.48853024840355, "requests_per_second": { "average": 27.31162397549392 }, "output_tokens_per_second": { "average": 873.9719672158054 }, "total_tokens_per_second": { "average": 2621.9159016474164 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 292.095058438008, "median": 291.98982599700685, "p99": 410.48853024840355 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0007/validation-001/attempts/001/repeats/003/short-chat/results.json", "repeat": 3, "elapsed_seconds": 6.7834368370022275, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000731", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.5347342940003728, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 29.921402422693404, "request_goodput": null, "output_throughput": 957.4848775261889, "total_token_throughput": "", "ttfts": [ 0.21156587500445312, 0.21757170699856943, 0.21729755999695044, 0.21723203799774637, 0.21718627699738136, 0.21677178500249283, 0.21672130799561273, 0.21650887199939461, 0.019857236002280843, 0.024550417001591995, 0.023913275996164884, 0.0239556239976082, 0.02390949200344039, 0.02349937500548549, 0.023466677994292695, 0.02344723799615167 ], "max_output_tokens_per_s": 498.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 119.84092237435107, "median_ttft_ms": 118.05814600302256, "std_ttft_ms": 96.52953006922047, "p50_ttft_ms": 118.05814600302256, "p95_ttft_ms": 217.3660967473552, "p99_ttft_ms": 217.53058494832658, "mean_tpot_ms": 4.741282487899416, "median_tpot_ms": 4.786395709684697, "std_tpot_ms": 0.1604404823516634, "p50_tpot_ms": 4.786395709684697, "p95_tpot_ms": 4.9167722177850415, "p99_tpot_ms": 5.021245049929054, "mean_itl_ms": 4.878999410766293, "median_itl_ms": 4.628522499842802, "std_itl_ms": 0.7949683066899527, "p50_itl_ms": 4.628522499842802, "p95_itl_ms": 6.614140148303704, "p99_itl_ms": 7.668258882476948, "mean_e2el_ms": 266.82067949923294, "median_e2el_ms": 266.5894345009292, "std_e2el_ms": 101.19627845043387, "p50_e2el_ms": 266.5894345009292, "p95_e2el_ms": 368.4305602491804, "p99_e2el_ms": 368.58714884692745, "requests_per_second": { "average": 29.921402422693404 }, "output_tokens_per_second": { "average": 957.4848775261889 }, "total_tokens_per_second": { "average": 2872.454632578567 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 266.82067949923294, "median": 266.5894345009292, "p99": 368.58714884692745 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0007/validation-001/attempts/001/repeats/004/short-chat/results.json", "repeat": 4, "elapsed_seconds": 6.7336320399990655, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000737", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.4735801179995178, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 33.78520210600625, "request_goodput": null, "output_throughput": 1081.1264673922, "total_token_throughput": "", "ttfts": [ 0.030154060004861094, 0.05216896800266113, 0.05190177600161405, 0.05181832499511074, 0.051737208996200934, 0.051297783000336494, 0.05107380600384204, 0.051147988000593614, 0.02188620899687521, 0.01686481700016884, 0.021426147999591194, 0.020923325006151572, 0.020881112999632023, 0.020857392002653796, 0.02043574499839451, 0.020396749998326413 ], "max_output_tokens_per_s": 498.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 34.685713375438354, "median_ttft_ms": 26.020134500868153, "std_ttft_ms": 15.120487545981986, "p50_ttft_ms": 26.020134500868153, "p95_ttft_ms": 51.96857400187582, "p99_ttft_ms": 52.12888920250407, "mean_tpot_ms": 6.493977671374112, "median_tpot_ms": 6.507895564526177, "std_tpot_ms": 1.7958590768549143, "p50_tpot_ms": 6.507895564526177, "p95_tpot_ms": 8.373698112982016, "p99_tpot_ms": 8.801768499970127, "mean_itl_ms": 6.682602504114437, "median_itl_ms": 4.772474498167867, "std_itl_ms": 4.569298749386771, "p50_itl_ms": 4.772474498167867, "p95_itl_ms": 17.633499948351528, "p99_itl_ms": 19.406997725964175, "mean_e2el_ms": 235.99902118803584, "median_e2el_ms": 235.9965329997067, "std_e2el_ms": 69.64730199600672, "p50_e2el_ms": 235.9965329997067, "p95_e2el_ms": 306.19241225394944, "p99_e2el_ms": 306.29962565326423, "requests_per_second": { "average": 33.78520210600625 }, "output_tokens_per_second": { "average": 1081.1264673922 }, "total_tokens_per_second": { "average": 3243.3794021766003 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 235.99902118803584, "median": 235.9965329997067, "p99": 306.29962565326423 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0007/validation-001/attempts/001/repeats/001/longer-prompt/results.json", "repeat": 1, "elapsed_seconds": 6.903703257004963, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000751", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8064904870043392, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 19.83904368101234, "request_goodput": null, "output_throughput": 1269.6987955847897, "total_token_throughput": "", "ttfts": [ 0.0233971669949824, 0.03764358299667947, 0.03727824599627638, 0.03726795500551816, 0.03718900400417624, 0.03666189999785274, 0.036616525998397265, 0.03656696299731266, 0.02414868000050774, 0.018934987994725816, 0.02368668100098148, 0.023115044998121448, 0.023056872996676248, 0.022704441005771514, 0.022633878004853614, 0.022608041996136308 ], "max_output_tokens_per_s": 1010.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 28.969373249310593, "median_ttft_ms": 23.91768050074461, "std_ttft_ms": 7.193985654265436, "p50_ttft_ms": 23.91768050074461, "p95_ttft_ms": 37.369580246377154, "p99_ttft_ms": 37.58878244661901, "mean_tpot_ms": 5.928139236121274, "median_tpot_ms": 5.944790119096906, "std_tpot_ms": 1.2273510034069401, "p50_tpot_ms": 5.944790119096906, "p95_tpot_ms": 7.185000194473856, "p99_tpot_ms": 7.322105194532289, "mean_itl_ms": 6.011635609655616, "median_itl_ms": 4.7814144963922445, "std_itl_ms": 3.5345839505968253, "p50_itl_ms": 4.7814144963922445, "p95_itl_ms": 12.42306090243801, "p99_itl_ms": 14.340110761404503, "mean_e2el_ms": 402.4421451249509, "median_e2el_ms": 402.68186399771366, "std_e2el_ms": 83.59926166336392, "p50_e2el_ms": 402.68186399771366, "p95_e2el_ms": 486.386955004491, "p99_e2el_ms": 486.7567494020477, "requests_per_second": { "average": 19.83904368101234 }, "output_tokens_per_second": { "average": 1269.6987955847897 }, "total_tokens_per_second": { "average": 6348.4939779239485 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 402.4421451249509, "median": 402.68186399771366, "p99": 486.7567494020477 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0007/validation-001/attempts/001/repeats/002/longer-prompt/results.json", "repeat": 2, "elapsed_seconds": 7.0475387389960815, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000758", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.9054140079970239, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 17.67147388783562, "request_goodput": null, "output_throughput": 1130.9743288214797, "total_token_throughput": "", "ttfts": [ 0.2733732750057243, 0.2792353670010925, 0.278873424998892, 0.27870120300212875, 0.27871503299684264, 0.2782281219988363, 0.2779937100058305, 0.27780275900295237, 0.024486410002282355, 0.01876921599614434, 0.02417157300078543, 0.024169655996956863, 0.02353931499965256, 0.023452673995052464, 0.023467130995413754, 0.02345092700124951 ], "max_output_tokens_per_s": 1010.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 150.5268622499898, "median_ttft_ms": 148.92984250400332, "std_ttft_ms": 127.3503098976611, "p50_ttft_ms": 148.92984250400332, "p95_ttft_ms": 278.96391049944214, "p99_ttft_ms": 279.18107570076245, "mean_tpot_ms": 4.785152301597658, "median_tpot_ms": 4.82108886509585, "std_tpot_ms": 0.06572928970428159, "p50_tpot_ms": 4.82108886509585, "p95_tpot_ms": 4.854625492029415, "p99_tpot_ms": 4.905430863569593, "mean_itl_ms": 4.852550318908514, "median_itl_ms": 4.856431500229519, "std_itl_ms": 0.5008024051925747, "p50_itl_ms": 4.856431500229519, "p95_itl_ms": 5.791837646756903, "p99_itl_ms": 6.36107291742519, "mean_e2el_ms": 451.99145725064227, "median_e2el_ms": 451.9123935024254, "std_e2el_ms": 123.64891847797949, "p50_e2el_ms": 451.9123935024254, "p95_e2el_ms": 576.3289942533447, "p99_e2el_ms": 576.3667444549355, "requests_per_second": { "average": 17.67147388783562 }, "output_tokens_per_second": { "average": 1130.9743288214797 }, "total_tokens_per_second": { "average": 5654.871644107399 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 451.99145725064227, "median": 451.9123935024254, "p99": 576.3667444549355 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0007/validation-001/attempts/001/repeats/003/longer-prompt/results.json", "repeat": 3, "elapsed_seconds": 6.956572656003118, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000805", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8072657140000956, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 19.819992008230024, "request_goodput": null, "output_throughput": 1268.4794885267215, "total_token_throughput": "", "ttfts": [ 0.028651449996687006, 0.05211753400362795, 0.05193730600149138, 0.051780825000605546, 0.05173463899700437, 0.0512014529958833, 0.05114497400063556, 0.05108670199842891, 0.024363035001442768, 0.017962332000024617, 0.024004652004805394, 0.023609545001818333, 0.02357781000318937, 0.023560482004540972, 0.02319338099914603, 0.023153066998929717 ], "max_output_tokens_per_s": 1010.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 35.817449188016326, "median_ttft_ms": 26.507242499064887, "std_ttft_ms": 14.027229468783064, "p50_ttft_ms": 26.507242499064887, "p95_ttft_ms": 51.98236300202552, "p99_ttft_ms": 52.090499803307466, "mean_tpot_ms": 5.827422376973522, "median_tpot_ms": 5.845571555601921, "std_tpot_ms": 0.9709497405206461, "p50_tpot_ms": 5.845571555601921, "p95_tpot_ms": 6.841007797548728, "p99_tpot_ms": 7.062952149955515, "mean_itl_ms": 5.909500505038365, "median_itl_ms": 4.901208001683699, "std_itl_ms": 3.911638354472403, "p50_itl_ms": 4.901208001683699, "p95_itl_ms": 16.65355999903113, "p99_itl_ms": 21.41598949820036, "mean_e2el_ms": 402.9450589373482, "median_e2el_ms": 402.99975750167505, "std_e2el_ms": 73.80023268549364, "p50_e2el_ms": 402.99975750167505, "p95_e2el_ms": 477.15354074534844, "p99_e2el_ms": 477.25069694788544, "requests_per_second": { "average": 19.819992008230024 }, "output_tokens_per_second": { "average": 1268.4794885267215 }, "total_tokens_per_second": { "average": 6342.397442633608 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 402.9450589373482, "median": 402.99975750167505, "p99": 477.25069694788544 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0007/validation-001/attempts/001/repeats/004/longer-prompt/results.json", "repeat": 4, "elapsed_seconds": 7.009083971002838, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000812", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8331021260019043, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 19.20532849530074, "request_goodput": null, "output_throughput": 1229.1410236992474, "total_token_throughput": "", "ttfts": [ 0.03459213599853683, 0.07524914700479712, 0.07483665999461664, 0.07466966699576005, 0.07432834800420096, 0.07416297800227767, 0.07418894000147702, 0.07379853900056332, 0.02393812900118064, 0.018309610000869725, 0.023493369997595437, 0.023528047997388057, 0.022949074998905417, 0.022920635004993528, 0.022465508998720907, 0.0224080359985237 ], "max_output_tokens_per_s": 1010.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 45.98992668752544, "median_ttft_ms": 29.265132499858737, "std_ttft_ms": 25.30026306455132, "p50_ttft_ms": 29.265132499858737, "p95_ttft_ms": 74.93978174716176, "p99_ttft_ms": 75.18727395327005, "mean_tpot_ms": 5.869199851177756, "median_tpot_ms": 5.862476198383356, "std_tpot_ms": 1.1160712757856863, "p50_tpot_ms": 5.862476198383356, "p95_tpot_ms": 7.05635863091124, "p99_tpot_ms": 7.444357669067805, "mean_itl_ms": 5.951866403412284, "median_itl_ms": 4.864535003434867, "std_itl_ms": 4.894480591370369, "p50_itl_ms": 4.864535003434867, "p95_itl_ms": 6.959942003959441, "p99_itl_ms": 27.547236770915333, "mean_e2el_ms": 415.74951731172405, "median_e2el_ms": 415.77060149938916, "std_e2el_ms": 93.1419055540832, "p50_e2el_ms": 415.77060149938916, "p95_e2el_ms": 509.6354880006402, "p99_e2el_ms": 509.68522080074763, "requests_per_second": { "average": 19.20532849530074 }, "output_tokens_per_second": { "average": 1229.1410236992474 }, "total_tokens_per_second": { "average": 6145.705118496237 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 415.74951731172405, "median": 415.77060149938916, "p99": 509.68522080074763 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] } ], "execution": { "mode": "sequential", "artifact_subdirectory": "validation-001" } }, { "trial_id": "trial-0003", "status": "completed", "failure": null, "benchmark_count": 8, "metrics": { "requests_per_second": { "average": 23.312843613922592 }, "throughput_tokens_per_second": { "average": 1051.9000374222219 }, "total_tokens_per_second": { "average": 4379.256279373461 }, "ttft_ms": { "average": 46.60568711733504, "median": 34.7600211871395, "p95": 74.46021050100171, "p99": 74.68792760055294 }, "tpot_ms": { "average": 6.8512268909674985, "median": 6.858569394703453, "p95": 9.081021743952505, "p99": 9.52731267396119 }, "itl_ms": { "average": 7.004936915760387, "median": 4.884500686785032, "p99": 26.17066222934227 }, "end_to_end_ms": { "average": 355.8646978047477, "median": 355.912043749413, "p95": 465.42979000014384, "p99": 465.52724289913385 }, "total_time_seconds": { "average": 0.7131386927485437 } }, "benchmarks": [ { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0003/validation-001/attempts/001/repeats/001/short-chat/results.json", "repeat": 1, "elapsed_seconds": 7.131891769000504, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000842", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.6129774839937454, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 26.102100677099678, "request_goodput": null, "output_throughput": 835.2672216671897, "total_token_throughput": "", "ttfts": [ 0.03933460600092076, 0.07421456599695375, 0.07380023899895605, 0.07370895100029884, 0.07371419200353557, 0.07321266099461354, 0.07308532500610454, 0.07297050600027433, 0.021973110000544693, 0.01707098400220275, 0.021665236999979243, 0.021143430007214192, 0.021084852000058163, 0.0207233629989787, 0.020671901002060622, 0.02064434099884238 ], "max_output_tokens_per_s": 498.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 44.93864150072113, "median_ttft_ms": 30.653858000732725, "std_ttft_ms": 25.618642295429026, "p50_ttft_ms": 30.653858000732725, "p95_ttft_ms": 73.90382074845547, "p99_ttft_ms": 74.1524169472541, "mean_tpot_ms": 8.410921562508813, "median_tpot_ms": 8.40170116122951, "std_tpot_ms": 3.688930826042196, "p50_tpot_ms": 8.40170116122951, "p95_tpot_ms": 12.235041225721943, "p99_tpot_ms": 12.912029741933258, "mean_itl_ms": 8.655225217823133, "median_itl_ms": 5.07533849668107, "std_itl_ms": 7.361915122395564, "p50_itl_ms": 5.07533849668107, "p95_itl_ms": 25.118609704441035, "p99_itl_ms": 26.51491699005419, "mean_e2el_ms": 305.67720993849434, "median_e2el_ms": 305.56845300088753, "std_e2el_ms": 138.3739080927413, "p50_e2el_ms": 305.56845300088753, "p95_e2el_ms": 444.7211727492686, "p99_e2el_ms": 444.82758575104526, "requests_per_second": { "average": 26.102100677099678 }, "output_tokens_per_second": { "average": 835.2672216671897 }, "total_tokens_per_second": { "average": 2505.8016650015693 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 305.67720993849434, "median": 305.56845300088753, "p99": 444.82758575104526 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0003/validation-001/attempts/001/repeats/002/short-chat/results.json", "repeat": 2, "elapsed_seconds": 6.632872797003074, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000849", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.5745625999988988, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 27.847270253982185, "request_goodput": null, "output_throughput": 891.1126481274299, "total_token_throughput": "", "ttfts": [ 0.13606986399827292, 0.17689162000169745, 0.17654367599607212, 0.17651734600076452, 0.1764947720002965, 0.17594807400018908, 0.17595768899627728, 0.17593450599815696, 0.02391258800344076, 0.0186865170035162, 0.02322814999934053, 0.023189908999484032, 0.022835086005216, 0.022770454997953493, 0.02278528299939353, 0.022753212004317902 ], "max_output_tokens_per_s": 498.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 96.90742168777433, "median_ttft_ms": 79.99122600085684, "std_ttft_ms": 74.98843211188255, "p50_ttft_ms": 79.99122600085684, "p95_ttft_ms": 176.63066199747846, "p99_ttft_ms": 176.83942840085365, "mean_tpot_ms": 6.117661157238599, "median_tpot_ms": 6.09880875790916, "std_tpot_ms": 1.5374906503461194, "p50_tpot_ms": 6.09880875790916, "p95_tpot_ms": 7.790082443710661, "p99_tpot_ms": 8.582633882378426, "mean_itl_ms": 6.29535520747746, "median_itl_ms": 4.93662749795476, "std_itl_ms": 5.761758715634968, "p50_itl_ms": 4.93662749795476, "p95_itl_ms": 27.473967645709973, "p99_itl_ms": 33.03753484295157, "mean_e2el_ms": 286.5549175621709, "median_e2el_ms": 286.59644499930437, "std_e2el_ms": 121.0620302960525, "p50_e2el_ms": 286.59644499930437, "p95_e2el_ms": 408.14197624968074, "p99_e2el_ms": 408.2474256512796, "requests_per_second": { "average": 27.847270253982185 }, "output_tokens_per_second": { "average": 891.1126481274299 }, "total_tokens_per_second": { "average": 2673.33794438229 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 286.5549175621709, "median": 286.59644499930437, "p99": 408.2474256512796 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0003/validation-001/attempts/001/repeats/003/short-chat/results.json", "repeat": 3, "elapsed_seconds": 6.497994081997604, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000855", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.5888842409985955, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 27.170025764092678, "request_goodput": null, "output_throughput": 869.4408244509657, "total_token_throughput": "", "ttfts": [ 0.028695707995211706, 0.05394537599931937, 0.05365088699909393, 0.0536077180004213, 0.05338163900159998, 0.053025421999336686, 0.0530341120029334, 0.05297087299550185, 0.022946624994801823, 0.01700898999843048, 0.022204559994861484, 0.022258849996433128, 0.021854006001376547, 0.021777592999569606, 0.021757583002909087, 0.021745973004726693 ], "max_output_tokens_per_s": 498.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 35.86661968665794, "median_ttft_ms": 25.821166495006764, "std_ttft_ms": 15.583324691418582, "p50_ttft_ms": 25.821166495006764, "p95_ttft_ms": 53.72450924915029, "p99_ttft_ms": 53.90120264928555, "mean_tpot_ms": 8.322444830663436, "median_tpot_ms": 8.34222579038807, "std_tpot_ms": 3.9117321936140077, "p50_tpot_ms": 8.34222579038807, "p95_tpot_ms": 12.337018419378682, "p99_tpot_ms": 12.82871718061459, "mean_itl_ms": 8.56417914730771, "median_itl_ms": 4.65833750058664, "std_itl_ms": 7.194066837138081, "p50_itl_ms": 4.65833750058664, "p95_itl_ms": 25.243688502450787, "p99_itl_ms": 28.93337915855227, "mean_e2el_ms": 293.8624094372244, "median_e2el_ms": 293.78420150169404, "std_e2el_ms": 135.53191970516784, "p50_e2el_ms": 293.78420150169404, "p95_e2el_ms": 430.05384049683926, "p99_e2el_ms": 430.16805289444164, "requests_per_second": { "average": 27.170025764092678 }, "output_tokens_per_second": { "average": 869.4408244509657 }, "total_tokens_per_second": { "average": 2608.322473352897 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 293.8624094372244, "median": 293.78420150169404, "p99": 430.16805289444164 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "short-chat", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0003/validation-001/attempts/001/repeats/004/short-chat/results.json", "repeat": 4, "elapsed_seconds": 6.58194099800312, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000902", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.5534208160024718, "completed": 16, "failed": 0, "total_input_tokens": 1024, "total_output_tokens": 512, "request_throughput": 28.911091772031458, "request_goodput": null, "output_throughput": 925.1549367050067, "total_token_throughput": "", "ttfts": [ 0.028658736999204848, 0.05360343599750195, 0.053213383005640935, 0.0530201140063582, 0.05300369200267596, 0.052648487995611504, 0.05263621999620227, 0.05226382600085344, 0.02375515900348546, 0.01830131399765378, 0.02306261500052642, 0.023021077002340462, 0.022637365997070447, 0.022592015004192945, 0.022548829998413566, 0.022538506003911607 ], "max_output_tokens_per_s": 498.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 36.09404862572774, "median_ttft_ms": 26.206948001345154, "std_ttft_ms": 14.951337303826998, "p50_ttft_ms": 26.206948001345154, "p95_ttft_ms": 53.31089625360619, "p99_ttft_ms": 53.544928048722795, "mean_tpot_ms": 7.74104331047408, "median_tpot_ms": 7.755184741859746, "std_tpot_ms": 3.240454127731893, "p50_tpot_ms": 7.755184741859746, "p95_tpot_ms": 11.082753854768066, "p99_tpot_ms": 11.56885825485299, "mean_itl_ms": 7.965890431527388, "median_itl_ms": 4.833781000343151, "std_itl_ms": 7.000572479656629, "p50_itl_ms": 4.833781000343151, "p95_itl_ms": 25.074097946708196, "p99_itl_ms": 30.39657281318796, "mean_e2el_ms": 276.0663912504242, "median_e2el_ms": 276.18325049843406, "std_e2el_ms": 114.059864671074, "p50_e2el_ms": 276.18325049843406, "p95_e2el_ms": 390.9021859981294, "p99_e2el_ms": 391.02895879987045, "requests_per_second": { "average": 28.911091772031458 }, "output_tokens_per_second": { "average": 925.1549367050067 }, "total_tokens_per_second": { "average": 2775.46481011502 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 276.0663912504242, "median": 276.18325049843406, "p99": 391.02895879987045 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "output": [ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0003/validation-001/attempts/001/repeats/001/longer-prompt/results.json", "repeat": 1, "elapsed_seconds": 6.788276153994957, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000916", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.747209738001402, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 21.41299716301339, "request_goodput": null, "output_throughput": 1370.431818432857, "total_token_throughput": "", "ttfts": [ 0.019600044994149357, 0.032816452003316954, 0.032355190996895544, 0.03242160200170474, 0.03228113799559651, 0.031904953000776004, 0.031908203003695235, 0.031839998002396896, 0.02716508899902692, 0.019248821001383476, 0.026831975003005937, 0.026355000998592004, 0.026306815998395905, 0.025942346997908317, 0.025882287001877557, 0.025806197998463176 ], "max_output_tokens_per_s": 1010.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 28.041632249824033, "median_ttft_ms": 26.99853200101643, "std_ttft_ms": 4.283640454422762, "p50_ttft_ms": 26.99853200101643, "p95_ttft_ms": 32.520314502107794, "p99_ttft_ms": 32.75722450307512, "mean_tpot_ms": 5.473858155775296, "median_tpot_ms": 5.51026929361609, "std_tpot_ms": 0.6573887449453857, "p50_tpot_ms": 5.51026929361609, "p95_tpot_ms": 6.156028043747822, "p99_tpot_ms": 6.283081100893527, "mean_itl_ms": 5.550956143874635, "median_itl_ms": 4.869499000051292, "std_itl_ms": 1.9919375005362796, "p50_itl_ms": 4.869499000051292, "p95_itl_ms": 10.738294400289305, "p99_itl_ms": 10.892562881999764, "mean_e2el_ms": 372.89469606366765, "median_e2el_ms": 373.058901001059, "std_e2el_ms": 43.86152897475845, "p50_e2el_ms": 373.058901001059, "p95_e2el_ms": 417.32829750253586, "p99_e2el_ms": 417.41385150089627, "requests_per_second": { "average": 21.41299716301339 }, "output_tokens_per_second": { "average": 1370.431818432857 }, "total_tokens_per_second": { "average": 6852.159092164285 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 372.89469606366765, "median": 373.058901001059, "p99": 417.41385150089627 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0003/validation-001/attempts/001/repeats/002/longer-prompt/results.json", "repeat": 2, "elapsed_seconds": 6.860419784999976, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000923", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8372651769968797, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 19.10983573613934, "request_goodput": null, "output_throughput": 1223.0294871129176, "total_token_throughput": "", "ttfts": [ 0.026970184000674635, 0.05408234400238143, 0.05378211900097085, 0.053717488000984304, 0.05352905500330962, 0.0530723219999345, 0.05301472499559168, 0.053027037996798754, 0.02653523499611765, 0.019521166999766137, 0.0262508939995314, 0.025783712000702508, 0.02572712500113994, 0.02535177300160285, 0.025297577005403582, 0.025247974001104012 ], "max_output_tokens_per_s": 1010.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 37.556920750375866, "median_ttft_ms": 26.752709498396143, "std_ttft_ms": 14.114723302663972, "p50_ttft_ms": 26.752709498396143, "p95_ttft_ms": 53.8571752513235, "p99_ttft_ms": 54.037310252169846, "mean_tpot_ms": 6.036659469243335, "median_tpot_ms": 6.053099396792407, "std_tpot_ms": 1.284336328233164, "p50_tpot_ms": 6.053099396792407, "p95_tpot_ms": 7.371543916676837, "p99_tpot_ms": 7.631143373839135, "mean_itl_ms": 6.121684336029128, "median_itl_ms": 4.930613500619074, "std_itl_ms": 4.339929577470708, "p50_itl_ms": 4.930613500619074, "p95_itl_ms": 20.880236697848886, "p99_itl_ms": 21.21032407398161, "mean_e2el_ms": 417.86646731270594, "median_e2el_ms": 417.97899949597195, "std_e2el_ms": 93.23986631383295, "p50_e2el_ms": 417.97899949597195, "p95_e2el_ms": 511.70825575354684, "p99_e2el_ms": 511.7983775529865, "requests_per_second": { "average": 19.10983573613934 }, "output_tokens_per_second": { "average": 1223.0294871129176 }, "total_tokens_per_second": { "average": 6115.147435564588 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 417.86646731270594, "median": 417.97899949597195, "p99": 511.7983775529865 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0003/validation-001/attempts/001/repeats/003/longer-prompt/results.json", "repeat": 3, "elapsed_seconds": 7.012332906997472, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000930", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.9639925049996236, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 16.597639418375195, "request_goodput": null, "output_throughput": 1062.2489227760125, "total_token_throughput": "", "ttfts": [ 0.07484108499920694, 0.0372748000008869, 0.07432027700269828, 0.07413756800087867, 0.07416848900174955, 0.07373699999880046, 0.0736936559987953, 0.07361246399523225, 0.017874427998322062, 0.024611340995761566, 0.024309048996656202, 0.024329952997504734, 0.02371803300047759, 0.0236577670002589, 0.02362817899847869, 0.023246211996593047 ], "max_output_tokens_per_s": 1010.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 46.32251881139382, "median_ttft_ms": 30.943070498324232, "std_ttft_ms": 24.74208335322222, "p50_ttft_ms": 30.943070498324232, "p95_ttft_ms": 74.45047900182544, "p99_ttft_ms": 74.76296379973064, "mean_tpot_ms": 6.90448567757769, "median_tpot_ms": 6.911855182503738, "std_tpot_ms": 1.866204328728323, "p50_tpot_ms": 6.911855182503738, "p95_tpot_ms": 8.840749051621911, "p99_tpot_ms": 9.1977738230771, "mean_itl_ms": 7.001733520118941, "median_itl_ms": 4.892942997685168, "std_itl_ms": 6.26065029875771, "p50_itl_ms": 4.892942997685168, "p95_itl_ms": 27.094746201328235, "p99_itl_ms": 28.57316003413871, "mean_e2el_ms": 481.30511649878827, "median_e2el_ms": 481.38658799871337, "std_e2el_ms": 140.38328722500472, "p50_e2el_ms": 481.38658799871337, "p95_e2el_ms": 622.3734755021724, "p99_e2el_ms": 622.411358294994, "requests_per_second": { "average": 16.597639418375195 }, "output_tokens_per_second": { "average": 1062.2489227760125 }, "total_tokens_per_second": { "average": 5311.244613880062 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 481.30511649878827, "median": 481.38658799871337, "p99": 622.411358294994 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] }, { "name": "longer-prompt", "backend": "vllm", "backend_version": "0.28.0", "raw_artifact": "/var/tmp/vllm-optimizer-demo/public/report-showcase/20260913-205443-614946/trials/trial-0003/validation-001/attempts/001/repeats/004/longer-prompt/results.json", "repeat": 4, "elapsed_seconds": 6.826329779003572, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0, "workloads": [ { "index": 0, "configuration": { "backend": "vllm", "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8 }, "metrics": { "date": "20260914-000936", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "tokenizer_id": "/var/tmp/vllm-optimizer-demo/models/opt-125m", "num_prompts": 16, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 8, "duration": 0.8267969809967326, "completed": 16, "failed": 0, "total_input_tokens": 4096, "total_output_tokens": 1024, "request_throughput": 19.351788126646813, "request_goodput": null, "output_throughput": 1238.514440105396, "total_token_throughput": "", "ttfts": [ 0.03787791699869558, 0.07756397600314813, 0.0771904440043727, 0.07703436400333885, 0.07702091700048186, 0.0766693590048817, 0.07664916899375385, 0.07630621700081974, 0.023547401004179846, 0.01836031700077001, 0.023165983002400026, 0.022787661000620574, 0.022747321003407706, 0.022364961994753685, 0.022312899003736675, 0.022284190999926068 ], "max_output_tokens_per_s": 1010.0, "max_concurrent_requests": 16, "rtfx": 0.0, "mean_ttft_ms": 47.11769362620544, "median_ttft_ms": 30.712659001437714, "std_ttft_ms": 26.563699974592467, "p50_ttft_ms": 30.712659001437714, "p95_ttft_ms": 77.28382700406655, "p99_ttft_ms": 77.50794620333181, "mean_tpot_ms": 5.8027409642587395, "median_tpot_ms": 5.795410833328903, "std_tpot_ms": 0.96343006222084, "p50_tpot_ms": 5.795410833328903, "p95_tpot_ms": 6.834956995994111, "p99_tpot_ms": 7.21426403410049, "mean_itl_ms": 5.884471321924706, "median_itl_ms": 4.8788655003590975, "std_itl_ms": 4.616146085595667, "p50_itl_ms": 4.8788655003590975, "p95_itl_ms": 7.616853851141057, "p99_itl_ms": 29.806847039872082, "mean_e2el_ms": 412.69037437450606, "median_e2el_ms": 412.7395114992396, "std_e2el_ms": 84.88512303151295, "p50_e2el_ms": 412.7395114992396, "p95_e2el_ms": 498.20911574897764, "p99_e2el_ms": 498.3223327475571, "requests_per_second": { "average": 19.351788126646813 }, "output_tokens_per_second": { "average": 1238.514440105396 }, "total_tokens_per_second": { "average": 6192.57220052698 }, "time_to_first_token_ms": "", "time_per_output_token_ms": "", "inter_token_latency_ms": "", "end_to_end_latency_ms": { "average": 412.69037437450606, "median": 412.7395114992396, "p99": 498.3223327475571 }, "request_totals": { "successful": 16, "errored": 0, "incomplete": 0 }, "request_total": 16, "requests": { "input": [ 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256 ], "output": [ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 ], "error_msg": {} } }, "successful_requests": 16, "failed_requests": 0, "errored_requests": 0, "incomplete_requests": 0, "total_requests": 16, "failure_percentage": 0.0 } ] } ], "execution": { "mode": "sequential", "artifact_subdirectory": "validation-001" } } ], "ranking": [ { "trial_id": "trial-0007", "score": 1082.2693392419908, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 16, "max-num-batched-tokens": 512 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } }, { "trial_id": "trial-0003", "score": 1055.5243499491773, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 16, "max-num-batched-tokens": 256 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } }, { "trial_id": "trial-0006", "score": 678.2258335851593, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 4, "max-num-batched-tokens": 512 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } }, { "trial_id": "trial-0002", "score": 649.1315026015792, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 4, "max-num-batched-tokens": 256 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } }, { "trial_id": "trial-0005", "score": 201.95689630636957, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 1, "max-num-batched-tokens": 512 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } }, { "trial_id": "trial-0001", "score": 195.86536468526614, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 1, "max-num-batched-tokens": 256 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } } ], "best": { "trial_id": "trial-0007", "score": 1082.2693392419908, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 16, "max-num-batched-tokens": 512 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } }, "baseline": { "trial_id": "baseline", "score": 194.34017230857918, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 1, "max-num-batched-tokens": 256 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } }, "improvement_percent": 456.89429847964266, "best_by_benchmark": { "short-chat": { "trial_id": "trial-0007", "score": 915.7284223709971, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 16, "max-num-batched-tokens": 512 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } }, "longer-prompt": { "trial_id": "trial-0007", "score": 1248.8102561129845, "successful_requests": 128, "errored_requests": 0, "incomplete_requests": 0, "excluded_workloads": 0, "error_rate": 0.0, "server_args": { "host": "127.0.0.1", "port": 8766, "dtype": "float16", "seed": 42, "tensor-parallel-size": 1, "gpu-memory-utilization": 0.4, "max-model-len": 512, "enforce-eager": true, "enable-chunked-prefill": true, "max-num-seqs": 16, "max-num-batched-tokens": 512 }, "server_env": { "CUDA_VISIBLE_DEVICES": "0", "VLLM_USE_V2_MODEL_RUNNER": "0", "VLLM_USE_FLASHINFER_SAMPLER": "0" } } }, "benchmark_order": [ "short-chat", "longer-prompt" ], "benchmark_policy": { "repeats": 4, "minimum_repeats": 4, "warmup_repeats": 1, "drift_threshold": 0.0, "maximum_failure_percentage": 0.0 } }