{ "profile": "gb10", "slo": { "ttft_ms": 1000.0, "tpot_ms": 100.0 }, "environment": { "sglang": "0.5.22", "sglang_kernel": "0.4.9", "flashinfer": "0.7.1", "torch": "2.14.1+cu130", "cuda": "13.0", "driver": "580.159.03", "gpu": "NVIDIA GB10", "compute_capability": "12.1", "gpu_memory_total_gib": 121.7, "gpu_memory_free_gib_at_start": 112.1, "python": "3.12.3", "machine": "aarch64", "profile": "gb10", "models": [ "Qwen/Qwen3-30B-A3B-Instruct-2507", "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "lmsys/SGLang-EAGLE3-Qwen3-30B-A3B-Instruct-2507-SpecForge-Nex" ], "hf_hub_offline": true }, "load": { "vllm": { "kv_pool_tokens": null, "points": [ { "x": 1.0, "output_throughput": { "median": 29.46, "min": 29.3, "max": 29.46, "n": 3 }, "median_ttft_ms": { "median": 257.0, "min": 250.67, "max": 303.4, "n": 3 }, "p99_ttft_ms": { "median": 276.14, "min": 275.44, "max": 455.88, "n": 3 }, "median_tpot_ms": { "median": 32.26, "min": 31.99, "max": 32.38, "n": 3 }, "p99_tpot_ms": { "median": 32.5, "min": 32.16, "max": 32.74, "n": 3 }, "p99_itl_ms": { "median": 34.64, "min": 34.42, "max": 42.18, "n": 3 }, "median_itl_ms": { "median": 32.29, "min": 32.13, "max": 32.33, "n": 3 }, "goodput_rps": { "median": 0.23, "min": 0.23, "max": 0.23, "n": 3 }, "request_throughput": { "median": 0.23, "min": 0.23, "max": 0.23, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 84.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 68.7, 68.7, 78.1, 97.7, 99.9, 201.6, 204.3, 234.7, 236.3, 238.3, 239.4, 241.7, 242.3, 246.6, 247.0, 247.2, 248.6, 248.6, 250.3, 252.7, 254.1, 254.5, 254.7, 257.6, 259.7, 259.9, 260.8, 264.7, 264.8, 265.0, 265.2, 267.4, 269.5, 270.1, 270.1, 270.4, 276.3, 277.7, 288.7, 300.1, 301.7, 302.5, 303.0, 303.8, 305.2, 308.4, 319.4, 335.1, 373.9, 380.6, 469.2 ], "tpot_quantiles_ms": [ 31.8, 31.8, 31.9, 31.9, 31.9, 31.9, 31.9, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.1, 32.1, 32.1, 32.1, 32.1, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.3, 32.3, 32.3, 32.3, 32.3, 32.3, 32.4, 32.4, 32.4, 32.5, 32.5, 32.5, 32.5, 32.5, 32.6, 32.6, 32.7, 32.7, 32.7 ] }, { "x": 2.0, "output_throughput": { "median": 44.69, "min": 44.48, "max": 44.95, "n": 3 }, "median_ttft_ms": { "median": 151.28, "min": 140.89, "max": 186.41, "n": 3 }, "p99_ttft_ms": { "median": 223.4, "min": 221.32, "max": 246.26, "n": 3 }, "median_tpot_ms": { "median": 44.0, "min": 43.99, "max": 44.08, "n": 3 }, "p99_tpot_ms": { "median": 45.51, "min": 45.03, "max": 45.75, "n": 3 }, "p99_itl_ms": { "median": 48.87, "min": 47.03, "max": 54.78, "n": 3 }, "median_itl_ms": { "median": 43.98, "min": 43.65, "max": 44.05, "n": 3 }, "goodput_rps": { "median": 0.35, "min": 0.35, "max": 0.35, "n": 3 }, "request_throughput": { "median": 0.35, "min": 0.35, "max": 0.35, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 79.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 66.7, 66.7, 81.7, 96.9, 106.1, 108.4, 111.5, 121.6, 122.1, 123.6, 124.3, 127.5, 128.9, 131.4, 131.8, 133.6, 134.2, 134.2, 134.3, 135.1, 136.0, 137.6, 139.6, 142.0, 143.3, 144.2, 151.5, 159.3, 160.8, 184.2, 185.3, 188.6, 190.4, 191.9, 191.9, 192.8, 197.4, 197.9, 198.0, 209.6, 209.7, 214.6, 215.2, 218.8, 219.7, 219.9, 219.9, 221.6, 224.2, 246.0, 246.3 ], "tpot_quantiles_ms": [ 40.4, 40.4, 40.6, 40.9, 40.9, 41.1, 41.3, 43.0, 43.2, 43.2, 43.3, 43.3, 43.4, 43.4, 43.5, 43.6, 43.8, 43.8, 43.8, 43.8, 43.8, 43.9, 43.9, 44.0, 44.0, 44.0, 44.1, 44.1, 44.2, 44.2, 44.2, 44.3, 44.3, 44.4, 44.4, 44.4, 44.5, 44.5, 44.5, 44.6, 44.6, 44.7, 44.8, 44.8, 44.9, 45.1, 45.2, 45.2, 45.2, 45.6, 45.9 ] }, { "x": 4.0, "output_throughput": { "median": 57.08, "min": 56.51, "max": 57.19, "n": 3 }, "median_ttft_ms": { "median": 250.06, "min": 239.92, "max": 256.56, "n": 3 }, "p99_ttft_ms": { "median": 383.94, "min": 285.68, "max": 385.59, "n": 3 }, "median_tpot_ms": { "median": 69.2, "min": 69.02, "max": 69.36, "n": 3 }, "p99_tpot_ms": { "median": 72.69, "min": 70.19, "max": 73.24, "n": 3 }, "p99_itl_ms": { "median": 76.02, "min": 75.56, "max": 76.9, "n": 3 }, "median_itl_ms": { "median": 69.0, "min": 68.83, "max": 69.19, "n": 3 }, "goodput_rps": { "median": 0.45, "min": 0.44, "max": 0.45, "n": 3 }, "request_throughput": { "median": 0.45, "min": 0.44, "max": 0.45, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 78.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 72.5, 72.5, 80.1, 99.9, 134.3, 157.6, 161.5, 165.4, 167.0, 179.9, 180.4, 184.2, 186.1, 222.9, 223.1, 223.3, 225.5, 225.5, 230.6, 230.8, 231.1, 234.5, 239.9, 240.0, 240.0, 244.9, 245.5, 254.6, 254.8, 256.4, 256.8, 265.8, 266.0, 266.8, 266.8, 267.3, 274.6, 274.7, 274.8, 285.2, 285.4, 285.7, 295.4, 329.7, 332.8, 333.1, 336.8, 348.4, 357.2, 388.7, 392.1 ], "tpot_quantiles_ms": [ 63.3, 63.3, 63.3, 63.3, 64.1, 65.0, 65.7, 66.2, 66.2, 66.6, 66.6, 66.6, 67.3, 67.4, 67.7, 67.7, 68.5, 68.5, 68.7, 68.7, 68.8, 68.8, 68.9, 68.9, 69.0, 69.0, 69.2, 69.2, 69.4, 69.4, 69.4, 69.6, 69.9, 69.9, 69.9, 70.0, 70.2, 70.2, 70.2, 70.4, 70.7, 70.7, 71.5, 71.5, 72.1, 72.1, 72.4, 72.4, 72.4, 72.8, 73.4 ] }, { "x": 8.0, "output_throughput": { "median": 75.73, "min": 71.8, "max": 78.68, "n": 3 }, "median_ttft_ms": { "median": 377.83, "min": 300.03, "max": 470.72, "n": 3 }, "p99_ttft_ms": { "median": 1180.6, "min": 1132.62, "max": 1285.58, "n": 3 }, "median_tpot_ms": { "median": 99.88, "min": 99.04, "max": 107.93, "n": 3 }, "p99_tpot_ms": { "median": 111.85, "min": 106.55, "max": 113.95, "n": 3 }, "p99_itl_ms": { "median": 115.25, "min": 115.22, "max": 116.69, "n": 3 }, "median_itl_ms": { "median": 99.3, "min": 99.11, "max": 107.12, "n": 3 }, "goodput_rps": { "median": 0.23, "min": 0.0, "max": 0.28, "n": 3 }, "request_throughput": { "median": 0.59, "min": 0.56, "max": 0.61, "n": 3 }, "completed": [ 32, 32, 32 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 84.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 68.3, 83.0, 176.1, 238.5, 296.5, 296.5, 296.7, 298.6, 298.6, 298.7, 298.8, 299.0, 299.5, 299.8, 318.4, 318.9, 319.3, 320.2, 320.8, 320.9, 320.9, 321.0, 324.2, 325.0, 400.8, 437.4, 454.2, 470.5, 513.3, 566.8, 653.5, 653.6, 676.9, 677.4, 718.7, 735.0, 735.6, 799.8, 859.3, 859.5, 1041.4, 1041.9, 1042.3, 1089.9, 1096.6, 1097.3, 1126.4, 1132.5, 1132.6, 1168.1, 1356.9 ], "tpot_quantiles_ms": [ 81.4, 81.4, 81.4, 83.9, 87.1, 97.7, 97.7, 97.8, 97.8, 97.8, 97.8, 98.2, 98.5, 98.5, 98.7, 98.7, 98.7, 99.6, 99.6, 100.8, 100.8, 100.9, 100.9, 102.4, 102.9, 102.9, 104.1, 105.0, 105.0, 106.0, 106.0, 106.1, 106.1, 106.7, 107.1, 107.1, 107.1, 107.6, 107.9, 107.9, 107.9, 107.9, 108.2, 109.2, 111.1, 111.1, 111.1, 111.1, 112.3, 112.4, 114.7 ] }, { "x": 16.0, "output_throughput": { "median": 101.07, "min": 100.09, "max": 103.88, "n": 3 }, "median_ttft_ms": { "median": 528.86, "min": 528.56, "max": 546.09, "n": 3 }, "p99_ttft_ms": { "median": 2027.23, "min": 1951.53, "max": 2070.96, "n": 3 }, "median_tpot_ms": { "median": 153.9, "min": 153.74, "max": 155.35, "n": 3 }, "p99_tpot_ms": { "median": 160.05, "min": 159.89, "max": 163.5, "n": 3 }, "p99_itl_ms": { "median": 228.01, "min": 179.41, "max": 234.6, "n": 3 }, "median_itl_ms": { "median": 152.1, "min": 147.33, "max": 152.36, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 0.79, "min": 0.78, "max": 0.81, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 86.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 72.9, 262.7, 300.9, 309.4, 309.6, 328.3, 329.5, 332.2, 346.0, 347.7, 355.4, 355.5, 355.6, 355.8, 380.6, 457.1, 466.3, 467.1, 512.8, 521.4, 522.3, 528.4, 528.5, 528.6, 528.9, 546.0, 546.1, 760.7, 824.9, 830.7, 835.6, 837.6, 842.6, 1107.7, 1108.6, 1163.1, 1230.5, 1270.9, 1314.8, 1316.5, 1328.8, 1334.7, 1343.8, 1510.5, 1623.5, 1624.9, 1658.8, 1789.3, 1951.8, 2027.4, 2071.0 ], "tpot_quantiles_ms": [ 134.2, 134.2, 134.2, 135.2, 135.4, 140.3, 141.3, 141.3, 141.3, 142.7, 145.9, 146.3, 147.3, 147.9, 148.5, 148.8, 150.5, 151.0, 151.7, 153.1, 153.9, 153.9, 153.9, 153.9, 153.9, 153.9, 153.9, 154.4, 154.6, 154.9, 154.9, 155.4, 155.4, 156.1, 156.3, 156.3, 156.4, 156.6, 156.8, 157.2, 157.7, 158.8, 158.8, 159.0, 159.0, 159.1, 159.9, 159.9, 161.1, 161.1, 165.3 ] }, { "x": 32.0, "output_throughput": { "median": 154.05, "min": 152.11, "max": 154.78, "n": 3 }, "median_ttft_ms": { "median": 643.79, "min": 624.89, "max": 683.67, "n": 3 }, "p99_ttft_ms": { "median": 3982.92, "min": 3964.86, "max": 3988.62, "n": 3 }, "median_tpot_ms": { "median": 198.24, "min": 194.12, "max": 199.94, "n": 3 }, "p99_tpot_ms": { "median": 215.92, "min": 205.63, "max": 219.34, "n": 3 }, "p99_itl_ms": { "median": 477.92, "min": 471.76, "max": 479.08, "n": 3 }, "median_itl_ms": { "median": 194.13, "min": 191.17, "max": 197.07, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 1.2, "min": 1.19, "max": 1.21, "n": 3 }, "completed": [ 128, 128, 128 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2405.0, "max": 2411.0, "n": 3 }, "max_temp_c": 88.0, "thermal_share": 0.017 }, "ttft_quantiles_ms": [ 93.4, 326.0, 373.6, 373.7, 419.2, 419.3, 502.8, 503.9, 509.1, 525.3, 526.1, 600.7, 612.0, 613.3, 614.5, 618.5, 618.7, 622.8, 630.6, 632.2, 635.6, 643.6, 664.4, 665.2, 667.0, 683.3, 728.9, 881.6, 942.2, 1148.9, 1303.1, 1343.2, 1379.3, 1392.4, 1409.9, 1426.7, 1440.6, 1454.9, 1472.8, 1743.3, 1853.3, 1879.4, 1928.8, 2106.1, 2335.6, 2587.4, 3037.6, 3139.4, 3553.2, 3965.0, 3988.7 ], "tpot_quantiles_ms": [ 185.6, 188.6, 188.6, 189.6, 189.6, 190.4, 190.8, 190.8, 190.9, 191.6, 191.8, 191.8, 191.8, 192.8, 193.2, 193.2, 193.4, 194.1, 194.1, 195.7, 196.9, 197.7, 198.2, 198.3, 198.7, 198.7, 198.8, 199.5, 199.7, 200.1, 201.2, 202.0, 203.5, 203.9, 204.2, 204.4, 204.7, 204.7, 204.7, 205.5, 205.5, 206.3, 206.7, 208.9, 210.7, 210.8, 210.8, 212.3, 212.4, 216.5, 221.1 ] }, { "x": 64.0, "output_throughput": { "median": 239.31, "min": 236.33, "max": 244.69, "n": 3 }, "median_ttft_ms": { "median": 964.79, "min": 799.86, "max": 976.24, "n": 3 }, "p99_ttft_ms": { "median": 8001.49, "min": 7989.22, "max": 8030.25, "n": 3 }, "median_tpot_ms": { "median": 243.77, "min": 239.42, "max": 245.24, "n": 3 }, "p99_tpot_ms": { "median": 275.38, "min": 268.14, "max": 284.43, "n": 3 }, "p99_itl_ms": { "median": 528.29, "min": 509.78, "max": 529.37, "n": 3 }, "median_itl_ms": { "median": 241.67, "min": 236.84, "max": 244.25, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 1.87, "min": 1.85, "max": 1.91, "n": 3 }, "completed": [ 256, 256, 256 ], "gpu": { "median_sm_mhz": { "median": 2398.0, "min": 2398.0, "max": 2411.0, "n": 3 }, "max_temp_c": 89.0, "thermal_share": 0.107 }, "ttft_quantiles_ms": [ 125.1, 381.9, 436.1, 572.4, 602.5, 605.8, 679.8, 684.6, 690.8, 702.4, 706.1, 713.4, 722.1, 724.8, 726.5, 739.4, 740.2, 743.1, 752.4, 758.2, 758.4, 773.2, 776.4, 776.5, 808.9, 949.2, 986.2, 1330.0, 1424.7, 1487.2, 1507.6, 1519.2, 1533.9, 1542.8, 1556.2, 1561.7, 1574.7, 1674.3, 1995.9, 2057.0, 2115.8, 2625.4, 3131.1, 3627.3, 4498.7, 5080.3, 5637.6, 6493.9, 7155.9, 7501.8, 8320.4 ], "tpot_quantiles_ms": [ 230.3, 236.3, 236.4, 236.4, 236.5, 237.3, 237.4, 237.4, 238.3, 238.3, 238.3, 238.6, 239.3, 239.4, 239.7, 239.7, 239.7, 239.9, 240.9, 242.6, 243.7, 243.8, 243.8, 244.2, 244.2, 244.3, 244.7, 245.2, 246.8, 250.5, 253.3, 257.3, 260.1, 260.7, 261.1, 261.2, 261.8, 264.0, 267.0, 270.0, 272.1, 272.4, 272.6, 272.8, 273.6, 274.0, 274.2, 274.3, 275.4, 280.3, 433.0 ] } ] }, "vllm-async": { "kv_pool_tokens": null, "points": [ { "x": 1.0, "output_throughput": { "median": 29.23, "min": 29.22, "max": 29.36, "n": 3 }, "median_ttft_ms": { "median": 255.39, "min": 252.39, "max": 302.5, "n": 3 }, "p99_ttft_ms": { "median": 284.69, "min": 276.99, "max": 505.12, "n": 3 }, "median_tpot_ms": { "median": 32.5, "min": 32.05, "max": 32.52, "n": 3 }, "p99_tpot_ms": { "median": 32.9, "min": 32.4, "max": 32.94, "n": 3 }, "p99_itl_ms": { "median": 34.9, "min": 34.72, "max": 42.51, "n": 3 }, "median_itl_ms": { "median": 32.47, "min": 32.2, "max": 32.54, "n": 3 }, "goodput_rps": { "median": 0.23, "min": 0.23, "max": 0.23, "n": 3 }, "request_throughput": { "median": 0.23, "min": 0.23, "max": 0.23, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2401.5, "max": 2411.0, "n": 3 }, "max_temp_c": 86.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 68.3, 68.3, 79.9, 97.2, 97.8, 198.1, 199.1, 231.9, 239.7, 241.3, 241.4, 247.0, 247.2, 247.7, 247.7, 248.0, 249.3, 249.3, 252.1, 252.7, 254.9, 255.2, 255.3, 255.6, 256.4, 257.7, 263.3, 263.5, 263.8, 265.2, 265.6, 268.0, 270.9, 271.0, 271.0, 278.1, 280.6, 285.9, 287.1, 292.6, 295.6, 297.8, 302.1, 302.9, 306.5, 313.9, 321.1, 356.4, 388.8, 410.3, 521.8 ], "tpot_quantiles_ms": [ 31.8, 31.8, 31.9, 31.9, 32.0, 32.0, 32.0, 32.0, 32.1, 32.1, 32.1, 32.1, 32.1, 32.1, 32.1, 32.2, 32.2, 32.2, 32.2, 32.2, 32.2, 32.3, 32.3, 32.3, 32.3, 32.3, 32.3, 32.3, 32.3, 32.3, 32.3, 32.4, 32.5, 32.5, 32.5, 32.5, 32.5, 32.7, 32.7, 32.8, 32.8, 32.8, 32.8, 32.8, 32.8, 32.8, 32.9, 32.9, 32.9, 32.9, 32.9 ] }, { "x": 2.0, "output_throughput": { "median": 44.16, "min": 44.05, "max": 44.49, "n": 3 }, "median_ttft_ms": { "median": 154.01, "min": 142.82, "max": 189.97, "n": 3 }, "p99_ttft_ms": { "median": 229.34, "min": 225.33, "max": 278.86, "n": 3 }, "median_tpot_ms": { "median": 44.43, "min": 44.42, "max": 44.7, "n": 3 }, "p99_tpot_ms": { "median": 45.82, "min": 45.64, "max": 46.12, "n": 3 }, "p99_itl_ms": { "median": 48.15, "min": 47.87, "max": 55.57, "n": 3 }, "median_itl_ms": { "median": 44.4, "min": 44.04, "max": 44.49, "n": 3 }, "goodput_rps": { "median": 0.35, "min": 0.34, "max": 0.35, "n": 3 }, "request_throughput": { "median": 0.35, "min": 0.34, "max": 0.35, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 79.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 67.9, 67.9, 79.9, 98.2, 109.9, 110.1, 111.3, 121.9, 122.6, 124.5, 126.7, 127.9, 128.6, 129.8, 134.0, 134.0, 134.8, 134.8, 134.9, 136.6, 137.4, 138.1, 140.9, 143.2, 145.9, 147.6, 150.9, 162.1, 165.3, 183.7, 188.2, 191.8, 192.9, 193.3, 193.3, 197.6, 198.5, 200.4, 201.1, 210.9, 212.9, 213.4, 216.5, 217.9, 222.4, 223.0, 223.0, 225.7, 230.6, 278.8, 278.9 ], "tpot_quantiles_ms": [ 40.8, 40.8, 41.2, 41.3, 41.4, 41.7, 41.9, 43.5, 43.5, 43.6, 43.8, 43.8, 43.9, 43.9, 44.0, 44.0, 44.1, 44.1, 44.2, 44.3, 44.3, 44.3, 44.4, 44.5, 44.5, 44.5, 44.5, 44.6, 44.6, 44.7, 44.7, 44.8, 44.8, 44.8, 44.8, 44.8, 44.8, 44.9, 45.0, 45.1, 45.1, 45.2, 45.2, 45.2, 45.5, 45.6, 45.7, 45.7, 45.7, 45.8, 46.2 ] }, { "x": 4.0, "output_throughput": { "median": 56.09, "min": 55.56, "max": 56.12, "n": 3 }, "median_ttft_ms": { "median": 255.87, "min": 242.88, "max": 267.2, "n": 3 }, "p99_ttft_ms": { "median": 356.93, "min": 321.72, "max": 388.31, "n": 3 }, "median_tpot_ms": { "median": 70.6, "min": 70.41, "max": 70.68, "n": 3 }, "p99_tpot_ms": { "median": 73.31, "min": 71.61, "max": 74.65, "n": 3 }, "p99_itl_ms": { "median": 77.46, "min": 76.6, "max": 77.91, "n": 3 }, "median_itl_ms": { "median": 70.39, "min": 69.91, "max": 70.57, "n": 3 }, "goodput_rps": { "median": 0.44, "min": 0.43, "max": 0.44, "n": 3 }, "request_throughput": { "median": 0.44, "min": 0.43, "max": 0.44, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 78.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 69.9, 69.9, 84.0, 101.1, 135.8, 160.8, 162.7, 168.9, 184.4, 187.9, 188.7, 194.1, 210.3, 224.6, 224.8, 224.9, 232.7, 232.7, 232.7, 232.9, 234.0, 237.4, 242.8, 243.0, 243.0, 250.5, 250.9, 257.9, 258.1, 260.8, 260.9, 267.1, 267.3, 268.0, 268.0, 286.5, 286.7, 287.2, 288.1, 288.2, 288.4, 321.5, 321.6, 321.7, 338.7, 339.2, 339.9, 344.5, 363.4, 369.0, 392.7 ], "tpot_quantiles_ms": [ 64.5, 64.5, 64.5, 64.5, 65.2, 66.2, 66.9, 67.4, 67.4, 68.0, 68.0, 68.0, 68.8, 69.0, 69.0, 69.0, 69.5, 69.5, 69.7, 70.0, 70.0, 70.0, 70.2, 70.2, 70.4, 70.4, 70.4, 70.4, 70.6, 70.6, 70.6, 70.8, 70.8, 71.2, 71.2, 71.2, 71.4, 71.7, 71.7, 71.7, 71.7, 71.8, 72.4, 72.8, 72.8, 73.4, 73.4, 73.8, 73.8, 73.8, 74.8 ] }, { "x": 8.0, "output_throughput": { "median": 74.89, "min": 71.05, "max": 77.81, "n": 3 }, "median_ttft_ms": { "median": 441.45, "min": 359.29, "max": 466.76, "n": 3 }, "p99_ttft_ms": { "median": 1141.49, "min": 1049.83, "max": 1171.7, "n": 3 }, "median_tpot_ms": { "median": 101.2, "min": 100.35, "max": 109.03, "n": 3 }, "p99_tpot_ms": { "median": 112.48, "min": 107.81, "max": 113.74, "n": 3 }, "p99_itl_ms": { "median": 117.34, "min": 116.69, "max": 117.66, "n": 3 }, "median_itl_ms": { "median": 100.88, "min": 100.03, "max": 108.6, "n": 3 }, "goodput_rps": { "median": 0.15, "min": 0.0, "max": 0.21, "n": 3 }, "request_throughput": { "median": 0.59, "min": 0.56, "max": 0.61, "n": 3 }, "completed": [ 32, 32, 32 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 84.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 71.9, 81.9, 202.1, 243.0, 291.5, 291.6, 301.1, 301.2, 301.3, 306.2, 306.3, 306.3, 306.4, 306.5, 306.7, 335.7, 336.1, 336.6, 337.3, 342.0, 342.4, 343.1, 386.4, 395.5, 419.9, 462.6, 463.0, 470.5, 480.0, 663.8, 668.1, 668.2, 681.4, 681.6, 743.4, 744.3, 772.6, 773.0, 781.7, 949.0, 949.6, 1049.7, 1049.8, 1058.1, 1058.8, 1092.1, 1096.4, 1118.7, 1141.4, 1162.1, 1176.0 ], "tpot_quantiles_ms": [ 82.1, 82.1, 82.1, 84.6, 87.9, 98.2, 98.2, 98.4, 99.2, 99.2, 99.2, 99.3, 99.3, 99.8, 99.9, 99.9, 100.0, 100.8, 100.8, 102.1, 102.1, 102.4, 102.4, 104.5, 105.4, 105.8, 105.8, 106.2, 106.6, 106.6, 107.2, 107.2, 107.3, 108.0, 108.7, 108.7, 108.7, 108.9, 109.0, 109.5, 109.5, 109.5, 109.5, 110.5, 110.9, 111.9, 111.9, 111.9, 112.1, 112.1, 114.5 ] }, { "x": 16.0, "output_throughput": { "median": 99.78, "min": 98.99, "max": 102.67, "n": 3 }, "median_ttft_ms": { "median": 555.17, "min": 538.81, "max": 557.04, "n": 3 }, "p99_ttft_ms": { "median": 2035.34, "min": 1970.89, "max": 2074.63, "n": 3 }, "median_tpot_ms": { "median": 156.3, "min": 155.72, "max": 157.32, "n": 3 }, "p99_tpot_ms": { "median": 161.75, "min": 161.2, "max": 165.47, "n": 3 }, "p99_itl_ms": { "median": 225.67, "min": 176.25, "max": 230.15, "n": 3 }, "median_itl_ms": { "median": 154.13, "min": 148.79, "max": 154.29, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 0.78, "min": 0.77, "max": 0.8, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 86.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 72.3, 309.2, 309.4, 314.5, 314.6, 317.8, 318.1, 349.2, 351.3, 387.7, 389.6, 419.6, 420.1, 472.4, 473.7, 517.4, 533.5, 533.7, 534.5, 534.7, 536.9, 538.6, 555.1, 555.1, 555.2, 556.5, 557.0, 787.9, 817.3, 825.7, 839.6, 843.1, 845.8, 1117.2, 1122.0, 1144.7, 1250.3, 1282.3, 1328.9, 1330.5, 1333.2, 1345.0, 1349.2, 1521.4, 1612.0, 1636.8, 1660.0, 1800.7, 1971.1, 2035.5, 2074.7 ], "tpot_quantiles_ms": [ 135.5, 135.5, 135.5, 136.5, 136.8, 141.6, 141.6, 142.5, 142.5, 144.7, 147.9, 148.1, 149.2, 149.8, 150.5, 150.7, 152.4, 153.0, 153.7, 155.0, 155.3, 155.3, 155.6, 155.9, 155.9, 156.3, 156.3, 156.3, 156.5, 156.8, 156.9, 157.3, 157.3, 158.1, 158.3, 158.3, 158.4, 158.5, 158.9, 159.1, 159.6, 159.9, 160.6, 160.9, 160.9, 161.0, 161.0, 161.5, 163.1, 163.2, 166.4 ] }, { "x": 32.0, "output_throughput": { "median": 150.2, "min": 148.09, "max": 150.74, "n": 3 }, "median_ttft_ms": { "median": 644.42, "min": 639.11, "max": 687.41, "n": 3 }, "p99_ttft_ms": { "median": 3979.68, "min": 3979.08, "max": 4005.81, "n": 3 }, "median_tpot_ms": { "median": 203.41, "min": 199.1, "max": 205.66, "n": 3 }, "p99_tpot_ms": { "median": 221.48, "min": 212.04, "max": 224.81, "n": 3 }, "p99_itl_ms": { "median": 477.02, "min": 473.45, "max": 479.31, "n": 3 }, "median_itl_ms": { "median": 199.37, "min": 196.89, "max": 203.15, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 1.17, "min": 1.16, "max": 1.18, "n": 3 }, "completed": [ 128, 128, 128 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 88.0, "thermal_share": 0.016 }, "ttft_quantiles_ms": [ 101.7, 332.8, 378.3, 378.5, 422.8, 423.0, 441.1, 505.7, 509.8, 522.6, 533.6, 534.6, 615.8, 616.5, 617.2, 617.6, 628.6, 634.0, 634.2, 635.9, 637.6, 643.8, 644.4, 676.2, 677.9, 685.8, 743.7, 887.8, 962.0, 1164.2, 1266.0, 1358.5, 1397.2, 1400.1, 1414.5, 1424.5, 1435.4, 1453.3, 1471.2, 1744.1, 1870.4, 1889.8, 1933.8, 2101.9, 2342.8, 2575.7, 3042.9, 3153.2, 3536.3, 3979.1, 4006.0 ], "tpot_quantiles_ms": [ 190.6, 194.0, 194.0, 195.1, 195.1, 195.8, 196.2, 196.2, 196.2, 197.0, 197.8, 197.8, 198.1, 198.2, 198.8, 198.8, 199.1, 199.1, 199.1, 200.7, 202.1, 203.1, 203.4, 203.4, 204.6, 204.6, 204.6, 205.6, 205.6, 205.7, 206.3, 207.1, 209.1, 209.3, 209.5, 209.6, 209.8, 211.1, 211.2, 211.2, 211.8, 212.0, 212.7, 214.0, 216.4, 216.5, 216.5, 217.8, 218.1, 222.0, 226.6 ] }, { "x": 64.0, "output_throughput": { "median": 235.9, "min": 233.2, "max": 240.16, "n": 3 }, "median_ttft_ms": { "median": 960.57, "min": 954.61, "max": 969.56, "n": 3 }, "p99_ttft_ms": { "median": 8035.29, "min": 7941.31, "max": 8045.03, "n": 3 }, "median_tpot_ms": { "median": 249.05, "min": 245.09, "max": 249.2, "n": 3 }, "p99_tpot_ms": { "median": 278.48, "min": 273.02, "max": 287.91, "n": 3 }, "p99_itl_ms": { "median": 527.43, "min": 523.6, "max": 528.94, "n": 3 }, "median_itl_ms": { "median": 245.81, "min": 241.22, "max": 247.93, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 1.84, "min": 1.82, "max": 1.88, "n": 3 }, "completed": [ 256, 256, 256 ], "gpu": { "median_sm_mhz": { "median": 2398.0, "min": 2398.0, "max": 2398.0, "n": 3 }, "max_temp_c": 89.0, "thermal_share": 0.108 }, "ttft_quantiles_ms": [ 115.4, 366.4, 388.8, 573.0, 637.8, 665.8, 667.4, 668.3, 683.3, 686.6, 686.9, 688.0, 689.6, 694.1, 705.2, 709.1, 712.4, 758.3, 758.4, 758.7, 759.6, 762.3, 775.8, 814.9, 953.1, 960.0, 988.2, 1204.0, 1425.4, 1515.2, 1530.8, 1537.9, 1543.2, 1551.8, 1561.1, 1570.6, 1578.9, 1666.1, 2033.1, 2066.4, 2112.3, 2625.5, 3139.2, 3645.4, 4609.5, 5124.5, 5639.1, 6664.5, 7193.6, 7727.7, 8348.3 ], "tpot_quantiles_ms": [ 234.1, 240.4, 240.4, 240.4, 240.5, 240.8, 240.8, 241.6, 241.8, 241.8, 243.5, 243.5, 243.6, 243.6, 244.6, 245.1, 245.1, 245.1, 245.8, 247.0, 248.1, 248.1, 248.1, 248.2, 248.3, 248.3, 249.1, 249.2, 250.4, 253.9, 257.1, 261.5, 264.2, 265.6, 266.1, 266.2, 266.3, 268.1, 270.8, 273.1, 275.4, 275.8, 275.9, 276.1, 276.8, 277.0, 277.3, 277.5, 278.5, 283.8, 440.3 ] } ] }, "vllm-eager": { "kv_pool_tokens": null, "points": [ { "x": 1.0, "output_throughput": { "median": 28.04, "min": 27.99, "max": 28.19, "n": 3 }, "median_ttft_ms": { "median": 248.81, "min": 247.02, "max": 306.73, "n": 3 }, "p99_ttft_ms": { "median": 273.7, "min": 271.76, "max": 472.03, "n": 3 }, "median_tpot_ms": { "median": 33.91, "min": 33.59, "max": 34.06, "n": 3 }, "p99_tpot_ms": { "median": 34.28, "min": 33.8, "max": 34.34, "n": 3 }, "p99_itl_ms": { "median": 36.73, "min": 36.28, "max": 43.36, "n": 3 }, "median_itl_ms": { "median": 33.97, "min": 33.73, "max": 34.02, "n": 3 }, "goodput_rps": { "median": 0.22, "min": 0.22, "max": 0.22, "n": 3 }, "request_throughput": { "median": 0.22, "min": 0.22, "max": 0.22, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2405.0, "max": 2411.0, "n": 3 }, "max_temp_c": 84.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 67.1, 67.1, 80.3, 84.4, 119.6, 193.9, 196.2, 228.0, 235.5, 236.2, 236.4, 236.6, 237.3, 241.9, 243.0, 244.2, 247.6, 247.6, 247.8, 248.4, 249.2, 249.8, 250.2, 252.2, 252.8, 257.3, 258.8, 263.2, 265.6, 266.8, 267.2, 267.9, 268.3, 269.9, 269.9, 270.5, 272.4, 274.4, 284.4, 289.3, 291.5, 293.1, 306.2, 307.3, 314.1, 319.1, 320.5, 336.3, 380.6, 401.0, 484.6 ], "tpot_quantiles_ms": [ 33.3, 33.3, 33.4, 33.5, 33.5, 33.5, 33.5, 33.6, 33.6, 33.6, 33.6, 33.7, 33.7, 33.7, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.8, 33.9, 33.9, 33.9, 33.9, 34.0, 34.0, 34.0, 34.0, 34.1, 34.1, 34.1, 34.2, 34.2, 34.2, 34.3, 34.3, 34.3, 34.3, 34.3, 34.3, 34.3, 34.3 ] }, { "x": 2.0, "output_throughput": { "median": 42.32, "min": 42.13, "max": 42.57, "n": 3 }, "median_ttft_ms": { "median": 144.82, "min": 144.48, "max": 189.52, "n": 3 }, "p99_ttft_ms": { "median": 227.68, "min": 226.98, "max": 283.58, "n": 3 }, "median_tpot_ms": { "median": 46.46, "min": 46.43, "max": 46.5, "n": 3 }, "p99_tpot_ms": { "median": 47.75, "min": 47.59, "max": 48.29, "n": 3 }, "p99_itl_ms": { "median": 50.57, "min": 49.77, "max": 53.4, "n": 3 }, "median_itl_ms": { "median": 46.43, "min": 46.25, "max": 46.68, "n": 3 }, "goodput_rps": { "median": 0.33, "min": 0.33, "max": 0.33, "n": 3 }, "request_throughput": { "median": 0.33, "min": 0.33, "max": 0.33, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 80.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 69.3, 69.3, 98.9, 111.5, 111.9, 112.2, 116.3, 123.8, 124.6, 127.7, 128.0, 128.7, 128.9, 132.3, 133.6, 135.1, 135.7, 135.7, 137.3, 137.5, 137.5, 137.8, 139.3, 141.3, 143.9, 145.7, 147.7, 152.0, 160.7, 164.4, 187.4, 191.7, 193.0, 194.9, 194.9, 198.1, 198.7, 199.4, 205.4, 210.1, 210.9, 211.5, 216.3, 217.1, 223.9, 224.7, 227.4, 228.4, 252.8, 257.5, 288.2 ], "tpot_quantiles_ms": [ 42.9, 42.9, 43.1, 43.3, 43.6, 43.7, 44.1, 45.6, 45.7, 45.7, 45.8, 45.9, 45.9, 45.9, 46.0, 46.1, 46.1, 46.1, 46.2, 46.3, 46.3, 46.3, 46.3, 46.4, 46.4, 46.5, 46.5, 46.5, 46.5, 46.6, 46.7, 46.7, 46.8, 46.9, 46.9, 47.0, 47.0, 47.1, 47.1, 47.1, 47.1, 47.2, 47.3, 47.3, 47.4, 47.6, 47.7, 47.7, 47.8, 47.8, 48.4 ] }, { "x": 4.0, "output_throughput": { "median": 55.33, "min": 54.89, "max": 55.35, "n": 3 }, "median_ttft_ms": { "median": 245.09, "min": 241.52, "max": 257.19, "n": 3 }, "p99_ttft_ms": { "median": 292.0, "min": 288.08, "max": 389.29, "n": 3 }, "median_tpot_ms": { "median": 71.4, "min": 71.33, "max": 71.75, "n": 3 }, "p99_tpot_ms": { "median": 74.65, "min": 72.52, "max": 75.75, "n": 3 }, "p99_itl_ms": { "median": 77.55, "min": 77.5, "max": 77.83, "n": 3 }, "median_itl_ms": { "median": 71.39, "min": 70.91, "max": 71.4, "n": 3 }, "goodput_rps": { "median": 0.43, "min": 0.43, "max": 0.43, "n": 3 }, "request_throughput": { "median": 0.43, "min": 0.43, "max": 0.43, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 79.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 70.8, 70.8, 84.7, 118.3, 141.4, 141.9, 154.1, 162.2, 167.6, 171.2, 191.3, 192.3, 201.5, 223.9, 224.2, 224.3, 232.0, 232.0, 232.2, 232.6, 235.1, 238.0, 241.4, 241.6, 242.0, 244.9, 245.2, 245.4, 254.2, 254.6, 259.7, 259.8, 260.7, 260.8, 260.8, 261.3, 279.3, 279.4, 279.6, 287.7, 287.9, 288.1, 291.5, 291.8, 292.0, 339.9, 340.3, 342.8, 343.7, 363.8, 393.8 ], "tpot_quantiles_ms": [ 65.4, 65.4, 65.4, 65.4, 66.2, 67.1, 67.8, 68.2, 68.3, 69.1, 69.1, 69.1, 69.8, 70.2, 70.2, 70.2, 71.0, 71.0, 71.0, 71.1, 71.1, 71.3, 71.3, 71.3, 71.3, 71.3, 71.4, 71.4, 71.4, 71.4, 71.4, 71.9, 72.0, 72.1, 72.1, 72.2, 72.3, 72.5, 72.5, 72.5, 72.6, 72.8, 73.5, 74.0, 74.0, 74.0, 74.8, 74.9, 74.9, 74.9, 75.9 ] }, { "x": 8.0, "output_throughput": { "median": 72.82, "min": 68.93, "max": 75.39, "n": 3 }, "median_ttft_ms": { "median": 336.54, "min": 309.46, "max": 467.49, "n": 3 }, "p99_ttft_ms": { "median": 1152.5, "min": 1084.81, "max": 1260.69, "n": 3 }, "median_tpot_ms": { "median": 104.7, "min": 103.22, "max": 112.68, "n": 3 }, "p99_tpot_ms": { "median": 116.11, "min": 111.2, "max": 118.06, "n": 3 }, "p99_itl_ms": { "median": 120.89, "min": 120.67, "max": 123.49, "n": 3 }, "median_itl_ms": { "median": 104.03, "min": 103.96, "max": 111.74, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.07, "n": 3 }, "request_throughput": { "median": 0.57, "min": 0.54, "max": 0.59, "n": 3 }, "completed": [ 32, 32, 32 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 84.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 75.5, 86.3, 183.1, 251.2, 289.8, 289.9, 290.1, 290.4, 298.3, 298.4, 301.7, 302.1, 302.8, 303.8, 308.7, 308.9, 309.2, 309.6, 330.1, 330.2, 330.3, 330.4, 344.6, 345.0, 345.7, 400.3, 443.1, 463.3, 471.9, 549.4, 660.6, 660.8, 680.6, 681.4, 704.8, 709.0, 709.8, 774.9, 1012.7, 1012.8, 1038.2, 1038.2, 1038.9, 1074.7, 1077.8, 1084.7, 1084.8, 1100.3, 1100.9, 1121.1, 1332.5 ], "tpot_quantiles_ms": [ 85.3, 85.3, 85.3, 87.5, 90.7, 100.9, 100.9, 102.2, 102.2, 102.2, 102.7, 102.8, 102.8, 103.1, 103.3, 103.5, 103.6, 103.6, 104.2, 105.7, 105.7, 105.7, 105.8, 107.4, 107.4, 107.8, 108.7, 109.6, 110.4, 110.5, 110.6, 110.6, 110.8, 111.4, 112.1, 112.1, 112.1, 112.1, 112.9, 113.2, 113.2, 113.2, 113.2, 113.5, 114.2, 115.3, 115.3, 115.3, 116.5, 116.6, 118.7 ] }, { "x": 16.0, "output_throughput": { "median": 98.83, "min": 97.72, "max": 101.14, "n": 3 }, "median_ttft_ms": { "median": 541.16, "min": 522.28, "max": 568.66, "n": 3 }, "p99_ttft_ms": { "median": 1989.44, "min": 1934.53, "max": 2052.99, "n": 3 }, "median_tpot_ms": { "median": 158.16, "min": 157.81, "max": 160.24, "n": 3 }, "p99_tpot_ms": { "median": 163.61, "min": 163.46, "max": 166.89, "n": 3 }, "p99_itl_ms": { "median": 216.47, "min": 174.68, "max": 238.98, "n": 3 }, "median_itl_ms": { "median": 155.96, "min": 151.12, "max": 157.0, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 0.77, "min": 0.76, "max": 0.79, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 86.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 71.5, 234.2, 270.3, 338.5, 338.7, 338.8, 339.0, 341.2, 341.8, 343.3, 350.0, 361.0, 361.1, 361.2, 361.4, 367.8, 368.5, 444.6, 485.0, 485.8, 501.0, 501.5, 501.9, 525.3, 541.3, 568.0, 568.6, 744.3, 774.0, 784.8, 813.6, 827.5, 831.3, 1076.3, 1086.3, 1096.1, 1202.3, 1220.5, 1249.8, 1308.7, 1320.1, 1328.1, 1331.8, 1543.1, 1561.1, 1647.1, 1659.9, 1801.1, 1934.6, 1989.6, 2053.1 ], "tpot_quantiles_ms": [ 138.3, 138.3, 138.3, 139.3, 139.7, 144.1, 145.1, 145.2, 145.2, 147.6, 150.7, 150.8, 150.8, 152.4, 152.5, 154.1, 154.9, 155.6, 157.1, 157.8, 157.8, 158.1, 158.1, 158.1, 158.1, 158.2, 158.2, 158.3, 158.7, 158.7, 158.7, 158.8, 159.9, 160.0, 160.2, 160.2, 160.2, 160.2, 160.4, 161.4, 161.8, 162.4, 162.4, 162.4, 162.5, 162.6, 162.8, 163.3, 164.1, 166.0, 168.3 ] }, { "x": 32.0, "output_throughput": { "median": 149.13, "min": 131.41, "max": 150.85, "n": 3 }, "median_ttft_ms": { "median": 658.68, "min": 655.28, "max": 1350.0, "n": 3 }, "p99_ttft_ms": { "median": 3956.4, "min": 3910.29, "max": 16502.95, "n": 3 }, "median_tpot_ms": { "median": 204.19, "min": 203.48, "max": 205.29, "n": 3 }, "p99_tpot_ms": { "median": 225.2, "min": 210.95, "max": 319.35, "n": 3 }, "p99_itl_ms": { "median": 473.56, "min": 465.28, "max": 473.6, "n": 3 }, "median_itl_ms": { "median": 198.88, "min": 197.29, "max": 201.51, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 1.17, "min": 1.03, "max": 1.18, "n": 3 }, "completed": [ 128, 128, 128 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 87.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 103.9, 392.1, 392.8, 393.4, 394.0, 394.6, 394.7, 395.0, 498.5, 499.2, 500.5, 502.1, 503.2, 506.7, 509.7, 564.5, 632.4, 640.6, 641.6, 642.4, 643.4, 645.0, 655.2, 670.0, 825.4, 898.6, 1119.4, 1246.3, 1286.6, 1330.9, 1357.9, 1364.2, 1401.3, 1403.5, 1429.8, 1439.7, 1720.3, 1791.4, 1825.1, 1869.5, 2113.5, 2207.0, 2566.6, 2678.6, 3040.7, 3484.5, 3530.8, 3948.1, 16485.3, 16502.3, 16503.1 ], "tpot_quantiles_ms": [ 190.3, 193.7, 193.7, 193.7, 195.9, 196.4, 196.9, 197.1, 197.1, 197.1, 197.6, 197.6, 197.6, 198.0, 198.0, 198.0, 198.6, 199.5, 200.6, 202.0, 202.2, 203.0, 203.0, 203.6, 204.8, 205.2, 205.3, 205.5, 206.2, 206.6, 206.6, 206.9, 207.4, 208.3, 209.3, 209.5, 209.9, 210.5, 210.5, 210.5, 211.7, 213.3, 215.8, 216.3, 216.5, 216.7, 218.5, 221.0, 225.2, 319.3, 321.9 ] }, { "x": 64.0, "output_throughput": { "median": 237.5, "min": 232.86, "max": 239.96, "n": 3 }, "median_ttft_ms": { "median": 773.76, "min": 622.99, "max": 1002.64, "n": 3 }, "p99_ttft_ms": { "median": 7942.28, "min": 7795.61, "max": 8055.88, "n": 3 }, "median_tpot_ms": { "median": 248.0, "min": 245.06, "max": 249.73, "n": 3 }, "p99_tpot_ms": { "median": 277.61, "min": 272.66, "max": 287.43, "n": 3 }, "p99_itl_ms": { "median": 508.76, "min": 501.14, "max": 511.28, "n": 3 }, "median_itl_ms": { "median": 245.96, "min": 242.71, "max": 249.46, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 1.86, "min": 1.82, "max": 1.87, "n": 3 }, "completed": [ 256, 256, 256 ], "gpu": { "median_sm_mhz": { "median": 2398.0, "min": 2398.0, "max": 2411.0, "n": 3 }, "max_temp_c": 88.0, "thermal_share": 0.065 }, "ttft_quantiles_ms": [ 116.7, 401.9, 446.6, 446.9, 496.0, 496.5, 503.7, 504.1, 509.3, 510.7, 512.0, 514.1, 603.8, 608.4, 667.0, 700.9, 701.1, 733.2, 737.3, 740.1, 772.7, 773.4, 773.6, 774.3, 822.3, 926.6, 1002.6, 1191.4, 1364.0, 1439.5, 1470.3, 1484.0, 1499.4, 1506.8, 1510.3, 1519.5, 1527.9, 1653.6, 1952.7, 2006.8, 2045.1, 2599.2, 3235.0, 3710.4, 4471.2, 4986.4, 5513.9, 6462.6, 6962.8, 7553.4, 8095.3 ], "tpot_quantiles_ms": [ 235.6, 240.5, 240.5, 240.5, 240.5, 241.2, 241.8, 241.9, 242.9, 243.0, 243.7, 244.0, 244.0, 244.3, 244.4, 245.0, 245.1, 245.1, 245.1, 248.0, 248.0, 248.0, 248.0, 248.2, 248.6, 248.6, 249.5, 249.5, 252.3, 255.4, 258.2, 261.3, 264.6, 266.3, 266.4, 266.5, 266.7, 268.8, 270.4, 272.7, 275.2, 275.6, 275.7, 275.8, 275.9, 276.1, 276.3, 276.4, 278.2, 282.6, 442.2 ] } ] }, "sglang": { "kv_pool_tokens": 233303, "points": [ { "x": 1.0, "output_throughput": { "median": 29.35, "min": 29.1, "max": 29.76, "n": 3 }, "median_ttft_ms": { "median": 270.12, "min": 269.81, "max": 307.12, "n": 3 }, "p99_ttft_ms": { "median": 298.41, "min": 297.32, "max": 424.18, "n": 3 }, "median_tpot_ms": { "median": 32.01, "min": 31.94, "max": 32.71, "n": 3 }, "p99_tpot_ms": { "median": 32.22, "min": 32.15, "max": 33.11, "n": 3 }, "p99_itl_ms": { "median": 34.83, "min": 34.18, "max": 42.36, "n": 3 }, "median_itl_ms": { "median": 31.93, "min": 31.92, "max": 32.56, "n": 3 }, "goodput_rps": { "median": 0.23, "min": 0.23, "max": 0.23, "n": 3 }, "request_throughput": { "median": 0.23, "min": 0.23, "max": 0.23, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 79.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 41.6, 41.6, 41.7, 66.2, 93.4, 209.1, 210.0, 248.3, 249.5, 251.3, 252.1, 255.1, 260.3, 261.0, 262.3, 268.3, 268.4, 268.4, 268.4, 269.2, 270.4, 271.8, 272.3, 276.3, 276.6, 276.7, 276.8, 277.0, 278.4, 282.6, 283.0, 284.4, 285.6, 286.5, 286.5, 288.1, 288.7, 298.7, 298.9, 300.2, 300.8, 301.7, 307.0, 307.2, 307.8, 312.2, 314.1, 338.6, 354.7, 369.7, 433.8 ], "tpot_quantiles_ms": [ 31.6, 31.6, 31.6, 31.7, 31.8, 31.8, 31.8, 31.8, 31.9, 31.9, 31.9, 31.9, 31.9, 31.9, 31.9, 31.9, 31.9, 31.9, 31.9, 31.9, 32.0, 32.0, 32.0, 32.0, 32.0, 32.0, 32.1, 32.1, 32.1, 32.1, 32.1, 32.1, 32.1, 32.1, 32.1, 32.2, 32.2, 32.2, 32.2, 32.6, 32.7, 32.7, 32.7, 32.7, 32.7, 32.8, 32.8, 32.9, 32.9, 33.0, 33.1 ] }, { "x": 2.0, "output_throughput": { "median": 44.59, "min": 44.44, "max": 45.23, "n": 3 }, "median_ttft_ms": { "median": 72.21, "min": 68.79, "max": 73.83, "n": 3 }, "p99_ttft_ms": { "median": 82.13, "min": 76.07, "max": 85.58, "n": 3 }, "median_tpot_ms": { "median": 44.61, "min": 44.54, "max": 45.22, "n": 3 }, "p99_tpot_ms": { "median": 45.98, "min": 45.93, "max": 46.15, "n": 3 }, "p99_itl_ms": { "median": 49.09, "min": 48.46, "max": 55.67, "n": 3 }, "median_itl_ms": { "median": 44.83, "min": 44.42, "max": 45.07, "n": 3 }, "goodput_rps": { "median": 0.35, "min": 0.35, "max": 0.35, "n": 3 }, "request_throughput": { "median": 0.35, "min": 0.35, "max": 0.35, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 77.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 60.1, 60.1, 60.5, 60.8, 62.5, 62.6, 63.0, 63.3, 63.7, 63.9, 63.9, 64.0, 64.1, 64.2, 64.5, 64.6, 64.8, 64.8, 64.9, 65.0, 65.1, 67.0, 71.0, 71.3, 71.7, 72.7, 72.7, 72.8, 73.1, 73.1, 73.2, 73.4, 73.4, 73.7, 73.7, 73.7, 74.4, 74.8, 74.8, 75.0, 75.2, 75.3, 76.0, 76.3, 76.4, 76.5, 76.9, 77.3, 83.0, 83.0, 86.0 ], "tpot_quantiles_ms": [ 40.8, 40.8, 40.9, 41.7, 41.8, 42.5, 42.5, 43.8, 43.9, 43.9, 43.9, 44.0, 44.0, 44.0, 44.1, 44.1, 44.2, 44.2, 44.3, 44.4, 44.4, 44.5, 44.5, 44.6, 44.6, 44.7, 44.7, 44.7, 44.8, 44.8, 44.9, 44.9, 45.1, 45.2, 45.2, 45.3, 45.3, 45.3, 45.3, 45.4, 45.4, 45.4, 45.4, 45.4, 45.4, 45.9, 45.9, 45.9, 46.0, 46.1, 46.2 ] }, { "x": 4.0, "output_throughput": { "median": 57.69, "min": 57.55, "max": 58.06, "n": 3 }, "median_ttft_ms": { "median": 114.75, "min": 110.29, "max": 115.33, "n": 3 }, "p99_ttft_ms": { "median": 116.23, "min": 114.25, "max": 116.8, "n": 3 }, "median_tpot_ms": { "median": 69.5, "min": 69.31, "max": 69.73, "n": 3 }, "p99_tpot_ms": { "median": 72.27, "min": 70.44, "max": 73.45, "n": 3 }, "p99_itl_ms": { "median": 78.62, "min": 75.37, "max": 81.87, "n": 3 }, "median_itl_ms": { "median": 68.97, "min": 68.93, "max": 69.25, "n": 3 }, "goodput_rps": { "median": 0.45, "min": 0.45, "max": 0.45, "n": 3 }, "request_throughput": { "median": 0.45, "min": 0.45, "max": 0.45, "n": 3 }, "completed": [ 16, 16, 16 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 77.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 46.6, 46.6, 47.0, 47.8, 47.9, 48.1, 48.1, 48.5, 48.8, 49.2, 50.9, 50.9, 51.0, 108.0, 108.1, 108.2, 110.3, 110.3, 110.3, 110.4, 111.0, 111.0, 111.2, 112.4, 112.5, 112.5, 114.1, 114.2, 114.3, 114.3, 114.6, 114.6, 114.7, 114.9, 114.9, 115.1, 115.2, 115.5, 115.7, 115.7, 115.7, 115.7, 115.7, 115.9, 116.0, 116.0, 116.1, 116.2, 116.6, 116.7, 116.8 ], "tpot_quantiles_ms": [ 63.8, 63.8, 63.8, 63.8, 64.3, 65.2, 65.2, 65.2, 65.7, 66.6, 66.6, 66.6, 67.2, 67.9, 67.9, 67.9, 68.4, 68.4, 69.3, 69.3, 69.3, 69.3, 69.3, 69.3, 69.5, 69.5, 69.5, 69.7, 69.7, 69.7, 69.8, 69.8, 70.0, 70.0, 70.0, 70.0, 70.0, 70.2, 70.2, 70.2, 70.3, 70.5, 70.8, 71.9, 71.9, 71.9, 72.3, 73.0, 73.0, 73.0, 73.5 ] }, { "x": 8.0, "output_throughput": { "median": 75.52, "min": 72.28, "max": 78.68, "n": 3 }, "median_ttft_ms": { "median": 213.96, "min": 184.71, "max": 221.37, "n": 3 }, "p99_ttft_ms": { "median": 1086.01, "min": 970.22, "max": 1189.48, "n": 3 }, "median_tpot_ms": { "median": 101.26, "min": 99.28, "max": 108.89, "n": 3 }, "p99_tpot_ms": { "median": 112.83, "min": 106.98, "max": 113.32, "n": 3 }, "p99_itl_ms": { "median": 116.8, "min": 116.77, "max": 121.41, "n": 3 }, "median_itl_ms": { "median": 100.76, "min": 99.79, "max": 107.09, "n": 3 }, "goodput_rps": { "median": 0.17, "min": 0.0, "max": 0.41, "n": 3 }, "request_throughput": { "median": 0.59, "min": 0.56, "max": 0.61, "n": 3 }, "completed": [ 32, 32, 32 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 84.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 49.0, 49.4, 51.4, 64.5, 146.6, 147.0, 147.3, 148.0, 148.0, 148.1, 148.3, 148.9, 149.3, 149.4, 149.7, 158.9, 159.0, 159.1, 162.9, 163.1, 163.4, 163.6, 166.4, 166.5, 166.5, 166.6, 264.3, 285.7, 319.8, 615.9, 616.3, 949.5, 949.6, 949.7, 949.7, 969.5, 969.9, 970.1, 991.9, 992.3, 993.0, 993.2, 1032.4, 1032.8, 1033.2, 1033.4, 1085.9, 1086.0, 1189.0, 1189.2, 1189.5 ], "tpot_quantiles_ms": [ 82.3, 82.3, 82.3, 82.3, 89.4, 98.9, 98.9, 98.9, 98.9, 98.9, 98.9, 98.9, 99.3, 99.3, 99.3, 99.3, 99.6, 99.6, 99.6, 99.7, 102.2, 102.2, 102.2, 102.2, 103.4, 103.4, 104.3, 105.6, 105.6, 105.6, 105.6, 106.1, 106.1, 106.1, 106.1, 106.9, 107.1, 107.1, 108.9, 108.9, 108.9, 108.9, 109.0, 109.0, 109.0, 109.7, 112.2, 112.2, 112.2, 112.2, 114.7 ] }, { "x": 16.0, "output_throughput": { "median": 103.44, "min": 102.41, "max": 106.7, "n": 3 }, "median_ttft_ms": { "median": 257.58, "min": 252.87, "max": 276.73, "n": 3 }, "p99_ttft_ms": { "median": 1693.72, "min": 1643.27, "max": 1731.25, "n": 3 }, "median_tpot_ms": { "median": 152.17, "min": 149.59, "max": 154.34, "n": 3 }, "p99_tpot_ms": { "median": 158.01, "min": 155.49, "max": 161.61, "n": 3 }, "p99_itl_ms": { "median": 164.39, "min": 162.51, "max": 169.18, "n": 3 }, "median_itl_ms": { "median": 149.97, "min": 145.07, "max": 149.98, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 0.81, "min": 0.8, "max": 0.83, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 87.0, "thermal_share": 0.011 }, "ttft_quantiles_ms": [ 50.2, 53.0, 184.5, 184.7, 184.9, 184.9, 185.0, 185.1, 185.3, 196.8, 197.6, 197.8, 197.9, 198.1, 198.3, 199.0, 202.4, 202.5, 202.6, 202.7, 234.1, 234.4, 234.8, 235.4, 246.3, 249.9, 296.3, 619.1, 667.8, 678.8, 1461.4, 1462.2, 1462.9, 1463.6, 1642.6, 1642.8, 1643.1, 1643.3, 1670.4, 1671.3, 1672.7, 1685.4, 1686.4, 1687.4, 1691.3, 1691.6, 1692.7, 1693.8, 1729.4, 1730.2, 1731.3 ], "tpot_quantiles_ms": [ 132.9, 132.9, 132.9, 132.9, 134.0, 139.8, 139.8, 139.9, 139.9, 142.0, 142.0, 142.0, 142.1, 142.1, 142.1, 142.1, 145.8, 145.8, 145.8, 148.9, 149.0, 149.0, 149.0, 149.6, 149.6, 149.6, 149.6, 151.9, 151.9, 152.2, 152.2, 152.2, 152.4, 152.4, 152.4, 152.5, 152.5, 152.5, 152.8, 154.2, 154.3, 154.3, 154.4, 154.4, 157.2, 157.4, 157.6, 157.6, 157.6, 160.2, 163.6 ] }, { "x": 32.0, "output_throughput": { "median": 154.64, "min": 152.37, "max": 155.89, "n": 3 }, "median_ttft_ms": { "median": 400.92, "min": 350.71, "max": 430.56, "n": 3 }, "p99_ttft_ms": { "median": 3071.46, "min": 3030.63, "max": 3133.06, "n": 3 }, "median_tpot_ms": { "median": 195.94, "min": 194.32, "max": 200.27, "n": 3 }, "p99_tpot_ms": { "median": 219.65, "min": 209.43, "max": 224.67, "n": 3 }, "p99_itl_ms": { "median": 219.54, "min": 215.91, "max": 219.72, "n": 3 }, "median_itl_ms": { "median": 194.79, "min": 191.95, "max": 198.4, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 1.21, "min": 1.19, "max": 1.22, "n": 3 }, "completed": [ 128, 128, 128 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 88.0, "thermal_share": 0.042 }, "ttft_quantiles_ms": [ 53.3, 173.9, 177.5, 201.2, 228.9, 229.4, 230.6, 240.4, 240.6, 240.7, 241.6, 242.7, 250.4, 262.1, 265.6, 265.7, 265.7, 275.4, 324.5, 337.4, 337.5, 350.7, 371.5, 391.0, 400.0, 430.4, 656.0, 691.0, 1935.8, 1939.8, 1945.6, 1946.9, 1977.7, 1980.6, 1982.0, 1983.0, 1984.5, 1988.8, 1992.1, 2042.3, 2043.5, 2976.9, 3015.0, 3017.3, 3026.4, 3027.7, 3029.2, 3071.1, 3071.4, 3132.0, 3133.1 ], "tpot_quantiles_ms": [ 185.7, 185.7, 188.4, 188.4, 190.6, 190.6, 191.7, 191.7, 192.6, 193.1, 193.1, 193.4, 193.4, 193.4, 193.5, 194.1, 194.1, 194.3, 194.3, 194.8, 194.8, 194.8, 194.8, 195.2, 195.9, 196.0, 196.9, 196.9, 199.4, 200.3, 200.3, 200.3, 201.2, 201.2, 201.4, 201.4, 201.4, 201.4, 206.3, 206.3, 206.8, 207.6, 207.6, 208.7, 210.4, 210.4, 210.4, 214.4, 214.4, 214.5, 228.1 ] }, { "x": 64.0, "output_throughput": { "median": 245.09, "min": 239.36, "max": 247.7, "n": 3 }, "median_ttft_ms": { "median": 572.62, "min": 541.65, "max": 589.76, "n": 3 }, "p99_ttft_ms": { "median": 5560.9, "min": 5379.53, "max": 5750.67, "n": 3 }, "median_tpot_ms": { "median": 245.98, "min": 241.46, "max": 246.56, "n": 3 }, "p99_tpot_ms": { "median": 272.95, "min": 271.92, "max": 291.03, "n": 3 }, "p99_itl_ms": { "median": 270.76, "min": 267.51, "max": 273.3, "n": 3 }, "median_itl_ms": { "median": 239.3, "min": 237.22, "max": 244.16, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 1.91, "min": 1.87, "max": 1.94, "n": 3 }, "completed": [ 256, 256, 256 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2398.0, "max": 2411.0, "n": 3 }, "max_temp_c": 88.0, "thermal_share": 0.074 }, "ttft_quantiles_ms": [ 59.7, 193.6, 231.1, 232.2, 235.3, 274.7, 276.7, 277.1, 283.3, 294.2, 296.1, 371.7, 404.4, 417.1, 417.5, 429.3, 429.5, 501.8, 504.8, 508.0, 524.0, 525.0, 541.6, 556.6, 572.6, 589.6, 1562.3, 1599.2, 1611.3, 1614.5, 1843.6, 1943.8, 1953.7, 2833.5, 2881.1, 2889.5, 3164.7, 3198.5, 3266.9, 4103.7, 4144.2, 4166.8, 4487.8, 4497.7, 4599.4, 5149.6, 5330.4, 5379.0, 5427.9, 5561.1, 5751.1 ], "tpot_quantiles_ms": [ 233.0, 235.2, 235.2, 236.4, 236.5, 237.1, 237.5, 239.0, 239.0, 239.1, 239.1, 240.2, 240.2, 240.3, 241.4, 241.4, 241.5, 241.5, 241.6, 242.4, 242.8, 242.8, 243.5, 243.6, 244.6, 245.0, 245.6, 246.0, 246.0, 246.2, 246.6, 247.6, 247.6, 251.6, 251.8, 253.3, 253.4, 255.0, 256.3, 259.8, 261.5, 261.9, 263.9, 265.2, 266.3, 270.3, 271.9, 271.9, 274.4, 280.7, 294.2 ] } ] } }, "mixed": { "vllm": { "kv_pool_tokens": null, "points": [ { "x": 0.5, "output_throughput": { "median": 72.27, "min": 58.95, "max": 72.7, "n": 3 }, "median_ttft_ms": { "median": 1134.92, "min": 866.48, "max": 1171.3, "n": 3 }, "p99_ttft_ms": { "median": 2240.67, "min": 1761.05, "max": 2358.33, "n": 3 }, "median_tpot_ms": { "median": 148.52, "min": 102.41, "max": 188.15, "n": 3 }, "p99_tpot_ms": { "median": 181.12, "min": 141.42, "max": 223.46, "n": 3 }, "p99_itl_ms": { "median": 571.37, "min": 485.7, "max": 588.26, "n": 3 }, "median_itl_ms": { "median": 118.0, "min": 84.53, "max": 138.92, "n": 3 }, "goodput_rps": { "median": 0.01, "min": 0.01, "max": 0.14, "n": 3 }, "request_throughput": { "median": 0.43, "min": 0.4, "max": 0.45, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2398.0, "min": 2398.0, "max": 2398.0, "n": 3 }, "max_temp_c": 88.0, "thermal_share": 0.096 }, "ttft_quantiles_ms": [ 46.4, 75.5, 484.5, 547.2, 564.4, 582.0, 603.7, 644.5, 661.8, 700.8, 722.9, 726.3, 784.7, 835.9, 850.5, 862.9, 903.1, 911.2, 936.9, 947.5, 967.6, 981.0, 996.2, 1011.7, 1030.9, 1047.2, 1054.1, 1103.1, 1111.9, 1126.4, 1139.2, 1167.1, 1186.0, 1196.5, 1209.4, 1227.4, 1231.6, 1260.9, 1271.7, 1307.7, 1331.7, 1357.5, 1404.3, 1414.9, 1454.8, 1506.0, 1677.8, 1732.8, 1876.2, 2016.6, 2581.5 ], "tpot_quantiles_ms": [ 57.3, 68.5, 75.3, 76.6, 83.2, 89.0, 90.2, 93.3, 98.0, 100.3, 102.4, 107.8, 110.3, 111.4, 116.3, 118.2, 120.9, 121.6, 123.9, 125.9, 128.2, 130.1, 132.5, 136.0, 137.3, 140.0, 140.6, 143.6, 146.3, 147.4, 149.3, 151.3, 157.4, 162.8, 168.6, 170.9, 170.9, 172.2, 175.6, 176.9, 181.2, 182.5, 190.6, 192.3, 199.8, 202.4, 205.7, 206.5, 214.4, 216.6, 227.4 ] }, { "x": 1.0, "output_throughput": { "median": 107.83, "min": 96.86, "max": 112.84, "n": 3 }, "median_ttft_ms": { "median": 462.39, "min": 343.4, "max": 557.43, "n": 3 }, "p99_ttft_ms": { "median": 975.99, "min": 459.61, "max": 1025.77, "n": 3 }, "median_tpot_ms": { "median": 155.12, "min": 117.39, "max": 186.18, "n": 3 }, "p99_tpot_ms": { "median": 196.3, "min": 138.09, "max": 218.65, "n": 3 }, "p99_itl_ms": { "median": 417.45, "min": 180.99, "max": 432.05, "n": 3 }, "median_itl_ms": { "median": 153.11, "min": 119.76, "max": 174.88, "n": 3 }, "goodput_rps": { "median": 0.03, "min": 0.0, "max": 0.11, "n": 3 }, "request_throughput": { "median": 0.66, "min": 0.65, "max": 0.68, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2398.0, "max": 2411.0, "n": 3 }, "max_temp_c": 86.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 43.5, 76.8, 181.4, 190.7, 212.0, 231.6, 235.5, 255.0, 270.1, 275.5, 292.3, 300.4, 311.3, 318.3, 320.0, 326.6, 331.8, 341.1, 353.0, 361.1, 366.0, 371.8, 373.3, 376.7, 386.3, 397.9, 420.4, 426.0, 428.5, 437.9, 449.2, 460.9, 481.9, 486.5, 504.0, 530.2, 555.1, 559.8, 590.1, 625.5, 631.4, 641.6, 674.2, 694.1, 732.6, 746.2, 779.8, 793.0, 857.6, 924.4, 1037.1 ], "tpot_quantiles_ms": [ 59.2, 76.9, 82.9, 97.2, 102.2, 107.7, 114.4, 115.4, 115.6, 116.2, 117.8, 119.5, 121.0, 123.5, 126.0, 127.4, 128.2, 129.8, 131.9, 132.6, 134.0, 135.9, 136.4, 138.4, 149.2, 151.7, 152.9, 154.8, 155.1, 164.3, 165.2, 167.6, 169.9, 173.0, 173.6, 176.9, 177.0, 179.2, 180.7, 183.2, 185.2, 187.8, 192.1, 193.9, 197.8, 200.4, 203.0, 209.4, 213.1, 216.1, 219.8 ] }, { "x": 2.0, "output_throughput": { "median": 128.76, "min": 123.2, "max": 135.73, "n": 3 }, "median_ttft_ms": { "median": 584.05, "min": 495.38, "max": 623.79, "n": 3 }, "p99_ttft_ms": { "median": 1407.89, "min": 1017.1, "max": 1493.09, "n": 3 }, "median_tpot_ms": { "median": 204.66, "min": 180.32, "max": 227.1, "n": 3 }, "p99_tpot_ms": { "median": 257.03, "min": 220.01, "max": 272.99, "n": 3 }, "p99_itl_ms": { "median": 589.77, "min": 474.32, "max": 615.19, "n": 3 }, "median_itl_ms": { "median": 196.16, "min": 162.38, "max": 215.88, "n": 3 }, "goodput_rps": { "median": 0.02, "min": 0.0, "max": 0.02, "n": 3 }, "request_throughput": { "median": 0.81, "min": 0.8, "max": 0.83, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 81.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 44.9, 74.6, 166.7, 188.5, 221.2, 232.7, 262.0, 293.1, 307.3, 313.1, 320.9, 338.5, 348.5, 367.6, 388.3, 398.6, 414.7, 425.7, 431.8, 452.3, 486.1, 489.0, 495.4, 520.4, 533.8, 588.0, 595.8, 605.5, 613.3, 620.9, 644.6, 702.1, 737.7, 757.4, 776.0, 795.4, 806.8, 830.8, 853.0, 861.6, 875.1, 925.4, 960.6, 993.9, 1012.4, 1028.7, 1154.2, 1226.1, 1287.4, 1358.2, 1612.7 ], "tpot_quantiles_ms": [ 95.9, 114.8, 127.4, 136.3, 146.4, 159.5, 163.3, 165.5, 167.8, 169.6, 173.1, 175.3, 180.1, 182.7, 187.0, 187.3, 188.7, 191.8, 193.0, 194.9, 195.9, 197.2, 199.1, 200.0, 201.6, 202.3, 203.8, 204.6, 205.4, 206.5, 209.6, 213.5, 215.5, 217.3, 219.3, 221.4, 223.3, 225.1, 225.4, 229.0, 233.7, 234.3, 235.0, 238.3, 242.7, 245.0, 249.6, 253.0, 256.2, 257.7, 277.8 ] } ] }, "vllm-mnbt-16384": { "kv_pool_tokens": null, "points": [ { "x": 0.5, "output_throughput": { "median": 72.49, "min": 59.33, "max": 73.06, "n": 3 }, "median_ttft_ms": { "median": 906.11, "min": 739.15, "max": 954.83, "n": 3 }, "p99_ttft_ms": { "median": 1707.19, "min": 1467.01, "max": 1744.51, "n": 3 }, "median_tpot_ms": { "median": 139.37, "min": 97.78, "max": 174.85, "n": 3 }, "p99_tpot_ms": { "median": 165.39, "min": 132.9, "max": 203.84, "n": 3 }, "p99_itl_ms": { "median": 797.81, "min": 684.88, "max": 799.38, "n": 3 }, "median_itl_ms": { "median": 114.01, "min": 81.66, "max": 130.08, "n": 3 }, "goodput_rps": { "median": 0.02, "min": 0.02, "max": 0.18, "n": 3 }, "request_throughput": { "median": 0.43, "min": 0.4, "max": 0.45, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2398.0, "min": 2398.0, "max": 2411.0, "n": 3 }, "max_temp_c": 89.0, "thermal_share": 0.121 }, "ttft_quantiles_ms": [ 44.9, 81.3, 489.3, 514.6, 555.5, 563.3, 582.1, 599.2, 609.8, 620.6, 629.0, 640.1, 663.4, 681.3, 698.3, 735.3, 751.9, 772.1, 783.8, 794.4, 805.7, 820.7, 824.3, 828.1, 842.5, 850.3, 874.9, 894.4, 914.2, 915.0, 931.4, 948.5, 974.2, 976.8, 980.7, 987.7, 1002.3, 1008.1, 1023.3, 1031.9, 1048.6, 1089.6, 1139.3, 1167.0, 1198.0, 1268.1, 1352.5, 1417.0, 1541.4, 1648.6, 1829.6 ], "tpot_quantiles_ms": [ 55.4, 66.5, 69.2, 73.3, 81.1, 83.6, 87.3, 90.6, 91.1, 97.3, 98.5, 103.0, 106.0, 107.7, 108.9, 112.9, 113.6, 118.1, 119.3, 120.5, 124.1, 125.7, 126.3, 127.0, 128.8, 131.1, 133.1, 133.7, 136.5, 137.1, 142.2, 144.3, 145.9, 152.3, 155.9, 159.0, 159.4, 160.8, 163.5, 165.0, 167.2, 173.2, 175.6, 180.9, 182.9, 185.6, 189.1, 189.6, 191.7, 193.6, 209.8 ] }, { "x": 1.0, "output_throughput": { "median": 107.22, "min": 96.57, "max": 113.2, "n": 3 }, "median_ttft_ms": { "median": 477.17, "min": 354.45, "max": 593.11, "n": 3 }, "p99_ttft_ms": { "median": 1008.74, "min": 493.96, "max": 1050.23, "n": 3 }, "median_tpot_ms": { "median": 156.87, "min": 118.89, "max": 187.7, "n": 3 }, "p99_tpot_ms": { "median": 197.3, "min": 141.76, "max": 220.5, "n": 3 }, "p99_itl_ms": { "median": 395.3, "min": 200.84, "max": 427.41, "n": 3 }, "median_itl_ms": { "median": 152.9, "min": 120.93, "max": 178.65, "n": 3 }, "goodput_rps": { "median": 0.03, "min": 0.0, "max": 0.11, "n": 3 }, "request_throughput": { "median": 0.67, "min": 0.65, "max": 0.67, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2398.0, "max": 2411.0, "n": 3 }, "max_temp_c": 84.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 46.5, 74.7, 170.0, 202.0, 235.9, 244.7, 268.0, 277.9, 281.6, 287.9, 295.6, 310.8, 323.9, 326.1, 329.9, 340.0, 349.3, 352.1, 356.9, 361.1, 371.6, 377.3, 381.9, 386.9, 393.0, 398.1, 414.6, 422.5, 431.9, 436.4, 449.9, 468.4, 494.9, 533.0, 562.4, 568.8, 574.6, 595.8, 614.4, 623.9, 633.4, 646.1, 664.1, 676.6, 718.8, 730.7, 760.0, 830.7, 856.1, 961.3, 1073.6 ], "tpot_quantiles_ms": [ 59.6, 76.9, 83.2, 98.0, 103.1, 108.7, 115.7, 116.9, 117.2, 118.3, 119.3, 121.5, 123.3, 125.0, 128.0, 128.4, 129.6, 130.9, 133.3, 134.5, 134.9, 135.5, 139.3, 140.6, 149.2, 152.9, 153.9, 154.8, 156.5, 163.7, 166.4, 168.2, 169.9, 171.3, 175.5, 176.8, 179.2, 179.8, 182.0, 184.1, 186.3, 189.1, 192.7, 194.8, 199.5, 202.7, 204.7, 211.1, 213.0, 217.2, 220.8 ] }, { "x": 2.0, "output_throughput": { "median": 127.62, "min": 122.59, "max": 135.58, "n": 3 }, "median_ttft_ms": { "median": 594.51, "min": 477.64, "max": 620.91, "n": 3 }, "p99_ttft_ms": { "median": 1508.33, "min": 1063.32, "max": 1522.12, "n": 3 }, "median_tpot_ms": { "median": 205.06, "min": 182.17, "max": 229.36, "n": 3 }, "p99_tpot_ms": { "median": 256.41, "min": 224.75, "max": 275.72, "n": 3 }, "p99_itl_ms": { "median": 586.2, "min": 479.12, "max": 608.62, "n": 3 }, "median_itl_ms": { "median": 197.7, "min": 163.86, "max": 219.51, "n": 3 }, "goodput_rps": { "median": 0.02, "min": 0.0, "max": 0.02, "n": 3 }, "request_throughput": { "median": 0.8, "min": 0.8, "max": 0.82, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 81.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 45.3, 72.3, 169.4, 195.9, 218.9, 237.5, 266.5, 279.0, 298.5, 320.1, 334.1, 338.7, 348.5, 368.0, 377.4, 388.0, 409.7, 431.0, 437.4, 460.1, 472.1, 478.0, 479.2, 514.1, 526.2, 577.5, 599.4, 608.0, 620.9, 633.0, 644.6, 675.9, 713.7, 740.7, 756.4, 772.9, 789.5, 818.7, 843.9, 885.4, 903.8, 908.2, 928.9, 995.2, 1020.9, 1054.5, 1172.8, 1221.7, 1288.7, 1449.0, 1724.1 ], "tpot_quantiles_ms": [ 97.5, 115.9, 128.2, 139.9, 148.6, 160.1, 164.4, 166.9, 169.0, 172.9, 175.8, 179.4, 181.5, 183.0, 189.1, 189.6, 191.3, 193.5, 194.6, 196.6, 197.9, 199.8, 200.0, 202.3, 203.6, 204.2, 204.8, 206.6, 207.6, 207.8, 210.8, 215.0, 216.9, 219.2, 221.2, 224.1, 225.3, 226.8, 227.8, 228.2, 232.8, 234.2, 237.5, 239.4, 245.6, 248.2, 253.4, 256.0, 256.8, 260.6, 280.6 ] } ] }, "vllm-mnbt-2048": { "kv_pool_tokens": null, "points": [ { "x": 0.5, "output_throughput": { "median": 72.04, "min": 58.79, "max": 72.61, "n": 3 }, "median_ttft_ms": { "median": 1144.72, "min": 914.2, "max": 1150.62, "n": 3 }, "p99_ttft_ms": { "median": 2214.12, "min": 1767.44, "max": 2388.45, "n": 3 }, "median_tpot_ms": { "median": 149.28, "min": 104.77, "max": 194.72, "n": 3 }, "p99_tpot_ms": { "median": 182.17, "min": 147.6, "max": 226.39, "n": 3 }, "p99_itl_ms": { "median": 573.51, "min": 483.16, "max": 594.16, "n": 3 }, "median_itl_ms": { "median": 117.86, "min": 84.86, "max": 141.54, "n": 3 }, "goodput_rps": { "median": 0.01, "min": 0.01, "max": 0.12, "n": 3 }, "request_throughput": { "median": 0.43, "min": 0.4, "max": 0.45, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2398.0, "min": 2398.0, "max": 2398.0, "n": 3 }, "max_temp_c": 89.0, "thermal_share": 0.096 }, "ttft_quantiles_ms": [ 49.3, 74.0, 519.0, 530.2, 546.8, 576.1, 592.3, 607.9, 635.2, 671.7, 710.1, 754.6, 793.4, 830.5, 871.6, 884.6, 910.9, 919.3, 937.5, 968.2, 973.8, 989.1, 996.4, 1011.8, 1025.5, 1041.3, 1074.6, 1087.8, 1108.0, 1127.3, 1141.5, 1176.7, 1188.2, 1204.4, 1214.8, 1267.8, 1280.5, 1285.5, 1298.0, 1311.5, 1344.6, 1389.4, 1440.3, 1502.1, 1512.6, 1626.4, 1733.5, 1790.1, 1843.6, 1979.3, 2640.0 ], "tpot_quantiles_ms": [ 58.1, 69.8, 75.2, 79.0, 85.8, 89.6, 92.3, 94.2, 100.6, 102.6, 104.8, 108.4, 112.0, 115.7, 118.8, 120.3, 122.5, 123.4, 125.8, 127.2, 128.4, 132.4, 135.1, 137.7, 141.0, 141.5, 143.3, 146.2, 148.5, 149.0, 151.0, 154.5, 159.4, 164.6, 169.6, 172.0, 173.3, 175.2, 177.2, 179.5, 184.5, 191.8, 197.7, 200.6, 204.0, 207.5, 209.7, 214.2, 216.3, 220.0, 231.4 ] }, { "x": 1.0, "output_throughput": { "median": 107.78, "min": 97.44, "max": 112.95, "n": 3 }, "median_ttft_ms": { "median": 454.36, "min": 357.84, "max": 590.47, "n": 3 }, "p99_ttft_ms": { "median": 969.42, "min": 492.48, "max": 1052.18, "n": 3 }, "median_tpot_ms": { "median": 156.45, "min": 118.41, "max": 186.87, "n": 3 }, "p99_tpot_ms": { "median": 197.75, "min": 139.72, "max": 220.0, "n": 3 }, "p99_itl_ms": { "median": 405.49, "min": 186.82, "max": 426.77, "n": 3 }, "median_itl_ms": { "median": 153.66, "min": 118.02, "max": 177.95, "n": 3 }, "goodput_rps": { "median": 0.03, "min": 0.0, "max": 0.11, "n": 3 }, "request_throughput": { "median": 0.66, "min": 0.66, "max": 0.68, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2398.0, "max": 2411.0, "n": 3 }, "max_temp_c": 84.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 45.4, 74.6, 163.6, 200.0, 233.4, 243.9, 264.3, 276.2, 279.2, 285.8, 299.8, 316.3, 320.9, 323.1, 335.3, 347.3, 352.4, 355.9, 367.7, 376.6, 379.7, 386.9, 394.4, 403.0, 415.1, 421.1, 426.2, 431.8, 436.2, 445.6, 451.4, 466.7, 476.3, 529.1, 545.4, 564.0, 581.9, 596.2, 598.8, 607.6, 637.0, 647.8, 662.2, 680.4, 688.3, 701.6, 731.5, 783.1, 860.4, 971.1, 1088.8 ], "tpot_quantiles_ms": [ 59.7, 77.6, 83.9, 98.4, 102.1, 108.9, 114.7, 116.1, 116.3, 116.9, 118.8, 120.9, 121.5, 123.9, 125.7, 126.5, 128.0, 129.8, 131.5, 132.9, 134.1, 135.8, 137.6, 138.9, 150.0, 152.6, 154.3, 154.7, 156.0, 164.5, 165.4, 168.9, 170.3, 172.4, 174.2, 177.6, 178.8, 180.0, 182.0, 184.5, 186.2, 188.7, 193.9, 195.2, 198.1, 201.6, 203.6, 210.5, 215.3, 217.1, 221.2 ] }, { "x": 2.0, "output_throughput": { "median": 128.06, "min": 123.99, "max": 135.67, "n": 3 }, "median_ttft_ms": { "median": 559.77, "min": 490.56, "max": 654.24, "n": 3 }, "p99_ttft_ms": { "median": 1426.89, "min": 1033.63, "max": 1507.8, "n": 3 }, "median_tpot_ms": { "median": 205.45, "min": 179.16, "max": 229.32, "n": 3 }, "p99_tpot_ms": { "median": 256.26, "min": 224.43, "max": 271.77, "n": 3 }, "p99_itl_ms": { "median": 587.46, "min": 477.34, "max": 608.78, "n": 3 }, "median_itl_ms": { "median": 197.74, "min": 163.69, "max": 217.36, "n": 3 }, "goodput_rps": { "median": 0.02, "min": 0.0, "max": 0.02, "n": 3 }, "request_throughput": { "median": 0.8, "min": 0.8, "max": 0.83, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 81.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 45.4, 76.0, 169.0, 195.1, 215.5, 238.9, 275.9, 283.2, 311.9, 325.0, 335.0, 349.4, 360.7, 365.5, 379.2, 400.4, 409.2, 420.8, 436.1, 442.9, 458.4, 491.7, 499.8, 543.3, 547.5, 562.0, 593.1, 605.7, 629.8, 651.7, 656.8, 666.3, 682.9, 684.4, 698.4, 728.4, 746.1, 761.1, 798.2, 818.1, 849.0, 866.7, 881.3, 900.4, 926.4, 947.3, 1038.5, 1086.5, 1323.1, 1407.4, 1567.3 ], "tpot_quantiles_ms": [ 96.8, 115.7, 127.2, 137.0, 148.1, 157.3, 162.2, 165.1, 168.6, 170.0, 174.4, 176.8, 179.3, 181.8, 186.2, 188.6, 190.4, 192.3, 193.3, 195.2, 197.5, 199.3, 199.8, 201.8, 202.3, 203.9, 204.4, 205.6, 206.5, 207.8, 209.5, 213.0, 216.5, 219.1, 220.7, 222.1, 224.0, 225.4, 226.1, 228.0, 233.9, 234.5, 236.1, 239.1, 244.4, 245.7, 250.7, 253.3, 256.0, 259.3, 275.9 ] } ] }, "sglang": { "kv_pool_tokens": 251910, "points": [ { "x": 0.5, "output_throughput": { "median": 70.86, "min": 58.08, "max": 71.08, "n": 3 }, "median_ttft_ms": { "median": 841.12, "min": 807.48, "max": 914.14, "n": 3 }, "p99_ttft_ms": { "median": 1692.68, "min": 1675.46, "max": 2123.3, "n": 3 }, "median_tpot_ms": { "median": 170.07, "min": 125.45, "max": 205.85, "n": 3 }, "p99_tpot_ms": { "median": 196.34, "min": 164.52, "max": 234.8, "n": 3 }, "p99_itl_ms": { "median": 922.13, "min": 811.41, "max": 924.51, "n": 3 }, "median_itl_ms": { "median": 130.39, "min": 99.05, "max": 150.22, "n": 3 }, "goodput_rps": { "median": 0.01, "min": 0.01, "max": 0.04, "n": 3 }, "request_throughput": { "median": 0.42, "min": 0.39, "max": 0.45, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 87.0, "thermal_share": 0.028 }, "ttft_quantiles_ms": [ 44.9, 57.9, 510.4, 543.5, 562.2, 585.2, 595.3, 604.3, 619.4, 632.7, 640.3, 662.2, 684.7, 703.3, 712.8, 730.4, 753.7, 756.1, 763.3, 784.5, 796.4, 811.4, 821.1, 825.3, 843.3, 845.5, 857.9, 869.9, 879.4, 902.9, 912.2, 920.8, 939.8, 946.9, 956.2, 965.1, 978.6, 983.2, 1013.0, 1019.3, 1048.0, 1111.6, 1157.4, 1196.6, 1247.5, 1293.8, 1400.0, 1529.7, 1596.6, 1715.9, 2264.6 ], "tpot_quantiles_ms": [ 72.3, 82.1, 97.9, 100.6, 104.0, 107.0, 112.5, 113.9, 114.3, 116.7, 124.7, 126.3, 128.5, 131.0, 135.3, 137.4, 138.8, 141.7, 144.3, 146.3, 147.0, 149.7, 150.8, 156.7, 161.4, 161.9, 166.3, 167.2, 168.7, 169.2, 170.2, 173.2, 175.2, 182.1, 183.2, 185.5, 186.3, 187.9, 189.1, 190.5, 195.2, 200.2, 209.0, 212.1, 214.7, 216.6, 221.9, 223.4, 224.6, 230.9, 234.8 ] }, { "x": 1.0, "output_throughput": { "median": 108.84, "min": 95.61, "max": 113.69, "n": 3 }, "median_ttft_ms": { "median": 243.63, "min": 222.37, "max": 287.29, "n": 3 }, "p99_ttft_ms": { "median": 409.45, "min": 355.32, "max": 445.83, "n": 3 }, "median_tpot_ms": { "median": 161.9, "min": 130.42, "max": 179.03, "n": 3 }, "p99_tpot_ms": { "median": 182.87, "min": 149.81, "max": 197.9, "n": 3 }, "p99_itl_ms": { "median": 219.39, "min": 178.76, "max": 235.68, "n": 3 }, "median_itl_ms": { "median": 163.19, "min": 129.39, "max": 183.48, "n": 3 }, "goodput_rps": { "median": 0.01, "min": 0.0, "max": 0.08, "n": 3 }, "request_throughput": { "median": 0.67, "min": 0.64, "max": 0.68, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 77.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 45.9, 48.0, 80.6, 96.1, 106.2, 107.6, 111.1, 130.3, 145.3, 152.7, 175.2, 178.6, 185.5, 195.5, 199.1, 203.0, 211.4, 214.6, 218.0, 222.6, 225.1, 227.8, 231.0, 233.7, 238.6, 243.6, 250.2, 254.1, 259.5, 263.8, 265.6, 268.7, 275.5, 288.7, 294.4, 297.6, 299.2, 303.2, 308.4, 315.1, 320.2, 321.7, 326.0, 337.2, 344.8, 347.5, 369.2, 379.4, 395.8, 420.1, 451.6 ], "tpot_quantiles_ms": [ 62.2, 86.1, 92.6, 106.0, 109.2, 114.7, 120.0, 125.3, 127.4, 128.5, 129.8, 130.8, 131.4, 132.8, 134.3, 134.7, 136.1, 136.9, 137.9, 138.3, 139.3, 145.1, 147.0, 149.2, 150.1, 153.0, 158.0, 159.4, 160.6, 161.9, 162.4, 165.4, 167.2, 167.8, 171.0, 173.0, 174.2, 175.0, 175.2, 176.8, 178.3, 179.5, 182.1, 183.0, 185.5, 189.2, 190.4, 193.3, 194.3, 196.3, 198.0 ] }, { "x": 2.0, "output_throughput": { "median": 133.22, "min": 124.75, "max": 139.78, "n": 3 }, "median_ttft_ms": { "median": 238.32, "min": 235.6, "max": 282.7, "n": 3 }, "p99_ttft_ms": { "median": 451.51, "min": 387.27, "max": 457.62, "n": 3 }, "median_tpot_ms": { "median": 189.75, "min": 170.87, "max": 206.34, "n": 3 }, "p99_tpot_ms": { "median": 217.96, "min": 190.47, "max": 225.32, "n": 3 }, "p99_itl_ms": { "median": 260.62, "min": 233.67, "max": 285.54, "n": 3 }, "median_itl_ms": { "median": 196.02, "min": 167.41, "max": 216.1, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 0.84, "min": 0.82, "max": 0.84, "n": 3 }, "completed": [ 48, 48, 48 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 76.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 44.5, 48.9, 67.2, 83.5, 97.1, 109.9, 129.5, 132.1, 136.2, 156.6, 176.3, 180.4, 184.7, 189.3, 192.0, 196.3, 201.5, 204.3, 210.0, 215.2, 221.3, 225.8, 232.8, 235.8, 240.8, 243.6, 246.7, 250.8, 255.3, 258.3, 262.4, 269.4, 276.6, 281.1, 294.3, 304.8, 306.8, 309.8, 312.8, 324.6, 348.2, 355.7, 361.2, 367.4, 372.6, 382.7, 395.6, 405.3, 419.8, 445.5, 466.1 ], "tpot_quantiles_ms": [ 102.0, 122.4, 131.0, 136.7, 143.0, 149.5, 157.4, 159.8, 161.1, 164.6, 167.3, 168.0, 170.3, 172.1, 173.2, 173.3, 174.5, 176.6, 178.0, 178.4, 179.1, 180.1, 180.9, 184.0, 184.7, 185.7, 187.8, 188.5, 188.8, 190.1, 191.3, 193.1, 195.8, 199.5, 201.4, 203.0, 203.6, 204.9, 205.8, 206.5, 207.0, 207.9, 210.7, 211.8, 214.8, 215.6, 218.4, 220.2, 221.2, 222.5, 226.9 ] } ] } }, "waves": { "vllm": { "kv_pool_tokens": null, "points": [ { "x": 0.0, "output_throughput": { "median": 97.0, "min": 93.33, "max": 97.75, "n": 3 }, "median_ttft_ms": { "median": 475.52, "min": 449.72, "max": 482.87, "n": 3 }, "p99_ttft_ms": { "median": 513.34, "min": 482.01, "max": 515.56, "n": 3 }, "median_tpot_ms": { "median": 154.56, "min": 150.23, "max": 155.89, "n": 3 }, "p99_tpot_ms": { "median": 163.62, "min": 161.28, "max": 175.74, "n": 3 }, "p99_itl_ms": { "median": 183.13, "min": 180.62, "max": 183.5, "n": 3 }, "median_itl_ms": { "median": 153.58, "min": 149.86, "max": 155.8, "n": 3 }, "goodput_rps": { "median": 0.02, "min": 0.0, "max": 0.08, "n": 3 }, "request_throughput": { "median": 1.37, "min": 1.32, "max": 1.6, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 80.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 74.6, 162.1, 272.5, 318.2, 318.4, 318.5, 318.6, 394.5, 394.6, 394.8, 419.4, 434.1, 439.0, 444.6, 448.0, 449.4, 451.0, 451.9, 452.5, 453.9, 455.1, 457.3, 458.6, 459.0, 460.6, 463.8, 464.5, 465.9, 466.8, 468.8, 471.7, 475.1, 476.6, 478.5, 479.7, 481.8, 482.9, 484.9, 488.3, 489.4, 490.1, 491.2, 493.8, 501.6, 506.9, 508.7, 511.7, 511.8, 511.9, 512.2, 520.9 ], "tpot_quantiles_ms": [ 83.8, 106.2, 117.5, 125.6, 130.3, 138.0, 142.0, 146.6, 147.8, 148.7, 149.0, 149.1, 149.3, 149.7, 149.9, 150.2, 150.4, 150.7, 150.8, 151.6, 152.0, 152.4, 152.6, 153.2, 153.4, 153.7, 153.9, 154.5, 154.7, 155.1, 155.2, 155.6, 155.8, 156.3, 156.7, 157.0, 157.4, 157.7, 159.2, 159.5, 159.9, 160.0, 160.4, 161.0, 161.5, 162.1, 162.3, 162.9, 163.2, 163.7, 188.9 ] }, { "x": 0.5, "output_throughput": { "median": 99.02, "min": 98.9, "max": 99.7, "n": 3 }, "median_ttft_ms": { "median": 453.85, "min": 452.88, "max": 464.26, "n": 3 }, "p99_ttft_ms": { "median": 505.15, "min": 486.04, "max": 514.37, "n": 3 }, "median_tpot_ms": { "median": 152.73, "min": 151.21, "max": 156.82, "n": 3 }, "p99_tpot_ms": { "median": 157.71, "min": 157.45, "max": 159.97, "n": 3 }, "p99_itl_ms": { "median": 187.94, "min": 182.41, "max": 190.66, "n": 3 }, "median_itl_ms": { "median": 150.85, "min": 149.88, "max": 153.26, "n": 3 }, "goodput_rps": { "median": 0.02, "min": 0.0, "max": 0.03, "n": 3 }, "request_throughput": { "median": 1.03, "min": 1.0, "max": 1.05, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 83.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 64.1, 144.5, 299.7, 299.9, 308.9, 309.1, 309.2, 389.8, 389.9, 389.9, 391.9, 406.3, 411.0, 416.0, 420.8, 427.1, 430.3, 431.8, 434.3, 438.4, 444.6, 447.9, 452.3, 453.1, 453.8, 454.3, 455.8, 458.2, 459.4, 461.2, 463.6, 464.4, 465.4, 466.8, 468.9, 469.9, 473.6, 475.7, 477.6, 478.9, 480.9, 482.8, 484.2, 486.5, 490.1, 492.5, 496.3, 496.5, 500.1, 501.6, 535.3 ], "tpot_quantiles_ms": [ 83.6, 101.8, 111.9, 131.0, 134.3, 139.9, 140.6, 142.3, 143.3, 143.6, 144.2, 145.0, 145.2, 145.8, 146.1, 146.5, 146.8, 147.1, 147.5, 148.3, 148.8, 149.5, 150.3, 151.2, 151.3, 151.6, 151.9, 152.7, 153.0, 153.2, 153.5, 153.7, 153.9, 154.0, 154.2, 154.5, 155.1, 155.7, 156.0, 156.6, 156.9, 157.6, 158.8, 159.0, 159.1, 159.1, 159.3, 159.4, 159.5, 159.8, 161.0 ] }, { "x": 1.0, "output_throughput": { "median": 97.73, "min": 96.29, "max": 99.61, "n": 3 }, "median_ttft_ms": { "median": 1185.45, "min": 1178.96, "max": 1242.23, "n": 3 }, "p99_ttft_ms": { "median": 2048.53, "min": 1944.94, "max": 2510.33, "n": 3 }, "median_tpot_ms": { "median": 157.55, "min": 155.62, "max": 158.69, "n": 3 }, "p99_tpot_ms": { "median": 164.23, "min": 162.76, "max": 169.54, "n": 3 }, "p99_itl_ms": { "median": 394.38, "min": 389.29, "max": 401.94, "n": 3 }, "median_itl_ms": { "median": 152.76, "min": 148.23, "max": 153.37, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 0.76, "min": 0.75, "max": 0.78, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 87.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 86.1, 395.1, 476.8, 544.4, 562.5, 663.4, 700.6, 723.9, 773.0, 776.4, 783.0, 788.1, 789.2, 880.5, 906.9, 946.7, 948.3, 951.3, 972.8, 1012.1, 1037.3, 1041.4, 1083.9, 1162.4, 1179.2, 1184.6, 1186.1, 1194.3, 1197.6, 1202.3, 1223.0, 1237.3, 1241.5, 1254.9, 1258.3, 1262.2, 1271.8, 1289.1, 1314.6, 1321.6, 1345.4, 1347.7, 1406.8, 1407.0, 1652.1, 1714.3, 1768.0, 1819.0, 2048.5, 2193.3, 2510.4 ], "tpot_quantiles_ms": [ 139.3, 139.5, 141.7, 143.3, 143.7, 144.5, 144.5, 145.5, 146.2, 147.1, 149.2, 149.5, 149.7, 149.7, 149.8, 151.4, 152.6, 152.6, 153.8, 154.9, 156.0, 156.1, 156.6, 157.0, 157.3, 157.5, 157.6, 157.7, 158.2, 158.3, 158.7, 158.8, 159.2, 159.3, 159.8, 159.8, 160.4, 160.7, 160.9, 161.1, 161.4, 161.9, 162.1, 162.6, 163.3, 163.7, 164.6, 165.2, 165.7, 167.1, 170.3 ] } ] }, "vllm-async": { "kv_pool_tokens": null, "points": [ { "x": 0.0, "output_throughput": { "median": 96.66, "min": 92.94, "max": 97.49, "n": 3 }, "median_ttft_ms": { "median": 472.31, "min": 452.17, "max": 479.8, "n": 3 }, "p99_ttft_ms": { "median": 532.35, "min": 520.2, "max": 536.25, "n": 3 }, "median_tpot_ms": { "median": 155.48, "min": 149.89, "max": 156.26, "n": 3 }, "p99_tpot_ms": { "median": 165.25, "min": 158.1, "max": 170.55, "n": 3 }, "p99_itl_ms": { "median": 190.82, "min": 182.28, "max": 195.15, "n": 3 }, "median_itl_ms": { "median": 153.37, "min": 149.08, "max": 153.94, "n": 3 }, "goodput_rps": { "median": 0.02, "min": 0.0, "max": 0.05, "n": 3 }, "request_throughput": { "median": 1.37, "min": 1.32, "max": 1.59, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 80.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 71.1, 302.0, 302.1, 302.2, 324.4, 325.1, 325.8, 337.9, 338.0, 338.1, 427.9, 435.7, 438.5, 440.6, 445.1, 447.8, 451.5, 452.3, 454.9, 455.5, 455.9, 458.9, 460.3, 461.8, 464.2, 466.6, 467.7, 469.1, 469.4, 471.5, 472.5, 472.8, 474.1, 476.3, 478.0, 481.8, 486.3, 487.3, 489.7, 493.4, 494.6, 495.1, 497.5, 499.9, 500.4, 501.5, 502.4, 510.6, 520.0, 525.2, 536.3 ], "tpot_quantiles_ms": [ 85.2, 107.4, 118.2, 126.0, 130.2, 138.9, 141.7, 145.6, 147.0, 148.1, 148.4, 148.7, 148.9, 149.5, 149.9, 150.1, 150.4, 151.2, 151.7, 152.0, 152.4, 152.6, 153.0, 153.5, 153.8, 154.0, 154.3, 154.5, 155.2, 155.5, 155.6, 155.9, 156.0, 156.3, 156.7, 157.2, 157.6, 158.1, 158.9, 159.1, 159.8, 160.0, 160.6, 161.0, 161.3, 161.9, 162.3, 162.7, 163.6, 163.9, 174.9 ] }, { "x": 0.5, "output_throughput": { "median": 98.92, "min": 98.77, "max": 100.29, "n": 3 }, "median_ttft_ms": { "median": 459.48, "min": 458.08, "max": 471.31, "n": 3 }, "p99_ttft_ms": { "median": 507.33, "min": 493.04, "max": 519.53, "n": 3 }, "median_tpot_ms": { "median": 152.49, "min": 149.72, "max": 157.53, "n": 3 }, "p99_tpot_ms": { "median": 158.35, "min": 155.35, "max": 160.01, "n": 3 }, "p99_itl_ms": { "median": 183.05, "min": 182.31, "max": 193.45, "n": 3 }, "median_itl_ms": { "median": 151.24, "min": 149.04, "max": 153.15, "n": 3 }, "goodput_rps": { "median": 0.02, "min": 0.0, "max": 0.03, "n": 3 }, "request_throughput": { "median": 1.02, "min": 1.0, "max": 1.06, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 80.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 62.2, 246.8, 276.1, 276.4, 283.7, 284.0, 401.0, 407.8, 415.5, 424.7, 429.6, 431.4, 433.0, 434.9, 437.6, 441.8, 442.8, 446.2, 449.7, 453.9, 455.8, 457.8, 459.3, 459.5, 461.2, 462.2, 464.1, 465.0, 465.9, 468.6, 469.6, 472.1, 472.9, 476.0, 476.7, 478.4, 478.5, 478.6, 480.5, 483.8, 486.7, 489.3, 490.1, 492.5, 492.6, 492.7, 493.8, 494.5, 495.5, 505.8, 524.3 ], "tpot_quantiles_ms": [ 83.3, 102.4, 112.3, 129.6, 133.4, 139.7, 140.5, 142.0, 143.0, 143.4, 143.7, 144.4, 144.5, 145.3, 145.7, 146.1, 146.7, 146.9, 147.2, 147.7, 148.0, 148.8, 150.0, 150.5, 150.9, 151.2, 151.7, 152.3, 152.5, 152.9, 153.2, 153.3, 153.7, 153.8, 154.1, 154.3, 154.6, 155.0, 155.5, 155.9, 156.5, 157.7, 158.7, 159.0, 159.1, 159.2, 159.3, 159.4, 159.5, 160.0, 164.8 ] }, { "x": 1.0, "output_throughput": { "median": 97.71, "min": 96.85, "max": 99.71, "n": 3 }, "median_ttft_ms": { "median": 1212.24, "min": 1109.38, "max": 1245.28, "n": 3 }, "p99_ttft_ms": { "median": 1909.65, "min": 1907.12, "max": 2436.7, "n": 3 }, "median_tpot_ms": { "median": 157.94, "min": 155.61, "max": 158.83, "n": 3 }, "p99_tpot_ms": { "median": 164.32, "min": 162.3, "max": 168.22, "n": 3 }, "p99_itl_ms": { "median": 405.39, "min": 359.5, "max": 410.91, "n": 3 }, "median_itl_ms": { "median": 153.07, "min": 146.27, "max": 153.42, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 0.76, "min": 0.76, "max": 0.78, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 85.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 68.0, 398.6, 555.3, 563.0, 590.1, 617.7, 669.8, 714.1, 725.1, 752.0, 767.9, 777.9, 806.3, 831.6, 876.0, 959.7, 965.6, 967.4, 978.7, 998.5, 1043.2, 1047.1, 1053.0, 1070.1, 1108.7, 1147.3, 1181.6, 1201.4, 1228.9, 1236.8, 1245.5, 1247.5, 1275.8, 1310.0, 1310.2, 1313.4, 1317.7, 1320.6, 1327.2, 1364.9, 1399.9, 1436.2, 1494.4, 1499.2, 1564.0, 1585.6, 1787.6, 1907.1, 1909.7, 2093.0, 2436.7 ], "tpot_quantiles_ms": [ 139.6, 140.7, 141.7, 142.3, 143.8, 144.3, 144.3, 144.8, 146.3, 147.1, 147.7, 147.7, 148.0, 148.4, 149.9, 149.9, 151.3, 151.4, 153.1, 153.9, 155.1, 156.0, 156.2, 156.5, 156.6, 157.4, 157.7, 157.7, 157.7, 158.1, 158.3, 158.3, 158.8, 158.8, 159.0, 159.4, 159.6, 160.4, 160.8, 160.8, 161.6, 161.9, 162.3, 162.7, 162.9, 163.0, 163.6, 164.3, 165.5, 166.4, 169.1 ] } ] }, "sglang": { "kv_pool_tokens": 234958, "points": [ { "x": 0.0, "output_throughput": { "median": 96.6, "min": 93.38, "max": 96.96, "n": 3 }, "median_ttft_ms": { "median": 324.4, "min": 318.91, "max": 324.79, "n": 3 }, "p99_ttft_ms": { "median": 383.93, "min": 381.27, "max": 404.63, "n": 3 }, "median_tpot_ms": { "median": 155.55, "min": 152.43, "max": 158.58, "n": 3 }, "p99_tpot_ms": { "median": 166.69, "min": 157.2, "max": 170.89, "n": 3 }, "p99_itl_ms": { "median": 213.67, "min": 190.63, "max": 224.61, "n": 3 }, "median_itl_ms": { "median": 151.53, "min": 149.52, "max": 154.82, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.05, "n": 3 }, "request_throughput": { "median": 1.36, "min": 1.32, "max": 1.6, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 80.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 49.1, 71.3, 181.8, 195.4, 195.5, 195.6, 195.7, 196.6, 200.6, 200.7, 200.7, 200.9, 204.0, 208.0, 210.8, 211.1, 211.3, 211.4, 214.0, 222.3, 237.2, 254.3, 267.8, 308.4, 316.5, 320.7, 325.0, 327.9, 329.3, 330.0, 330.6, 332.2, 336.0, 337.8, 339.3, 342.0, 343.4, 344.6, 345.8, 346.9, 348.2, 349.9, 351.3, 355.3, 356.8, 357.9, 362.0, 369.0, 374.5, 382.0, 404.9 ], "tpot_quantiles_ms": [ 89.2, 113.2, 125.1, 129.5, 137.2, 142.4, 144.1, 146.1, 148.4, 149.1, 150.4, 150.7, 151.0, 151.5, 151.9, 152.2, 152.3, 152.6, 152.8, 153.5, 153.8, 154.0, 154.1, 154.3, 154.8, 155.0, 155.4, 155.5, 155.8, 156.3, 156.4, 156.8, 157.3, 157.7, 158.1, 158.3, 158.6, 159.0, 159.7, 160.0, 161.1, 161.3, 161.4, 161.6, 162.2, 162.5, 162.6, 163.0, 164.7, 167.5, 173.9 ] }, { "x": 0.5, "output_throughput": { "median": 99.46, "min": 98.52, "max": 100.1, "n": 3 }, "median_ttft_ms": { "median": 314.84, "min": 312.37, "max": 319.1, "n": 3 }, "p99_ttft_ms": { "median": 363.39, "min": 362.36, "max": 369.46, "n": 3 }, "median_tpot_ms": { "median": 153.37, "min": 150.17, "max": 155.21, "n": 3 }, "p99_tpot_ms": { "median": 157.5, "min": 156.07, "max": 160.3, "n": 3 }, "p99_itl_ms": { "median": 197.57, "min": 192.51, "max": 201.16, "n": 3 }, "median_itl_ms": { "median": 150.85, "min": 147.57, "max": 151.97, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.02, "n": 3 }, "request_throughput": { "median": 1.02, "min": 1.0, "max": 1.05, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 81.0, "thermal_share": 0.0 }, "ttft_quantiles_ms": [ 51.5, 73.1, 181.6, 193.5, 195.2, 195.3, 195.3, 195.4, 205.0, 205.4, 205.9, 206.8, 207.2, 207.7, 207.8, 207.9, 208.6, 214.4, 218.7, 225.0, 232.5, 296.5, 304.7, 310.3, 314.0, 315.5, 319.4, 322.1, 323.2, 323.9, 325.3, 326.4, 328.1, 331.8, 333.2, 333.7, 334.9, 336.3, 337.9, 338.5, 340.0, 343.2, 344.9, 347.6, 351.0, 352.8, 354.1, 357.9, 362.7, 367.5, 369.8 ], "tpot_quantiles_ms": [ 90.0, 105.7, 118.2, 133.5, 138.2, 141.4, 142.0, 143.4, 144.0, 144.5, 145.0, 145.8, 146.1, 146.7, 147.3, 147.7, 148.0, 148.1, 148.6, 149.0, 149.2, 149.3, 150.3, 150.7, 151.1, 151.3, 151.7, 152.3, 152.9, 153.3, 153.4, 153.5, 153.9, 154.0, 154.1, 154.2, 154.7, 154.9, 155.1, 155.4, 155.8, 156.0, 156.2, 157.0, 157.6, 157.9, 157.9, 158.1, 158.4, 158.6, 161.5 ] }, { "x": 1.0, "output_throughput": { "median": 101.44, "min": 100.28, "max": 104.19, "n": 3 }, "median_ttft_ms": { "median": 1275.38, "min": 1268.0, "max": 1626.6, "n": 3 }, "p99_ttft_ms": { "median": 1722.69, "min": 1719.0, "max": 1729.81, "n": 3 }, "median_tpot_ms": { "median": 149.7, "min": 147.69, "max": 152.29, "n": 3 }, "p99_tpot_ms": { "median": 160.33, "min": 159.35, "max": 164.69, "n": 3 }, "p99_itl_ms": { "median": 159.48, "min": 159.32, "max": 189.5, "n": 3 }, "median_itl_ms": { "median": 147.92, "min": 143.69, "max": 148.15, "n": 3 }, "goodput_rps": { "median": 0.0, "min": 0.0, "max": 0.0, "n": 3 }, "request_throughput": { "median": 0.79, "min": 0.78, "max": 0.81, "n": 3 }, "completed": [ 64, 64, 64 ], "gpu": { "median_sm_mhz": { "median": 2411.0, "min": 2411.0, "max": 2411.0, "n": 3 }, "max_temp_c": 87.0, "thermal_share": 0.011 }, "ttft_quantiles_ms": [ 50.8, 206.7, 268.9, 291.9, 613.0, 731.9, 818.5, 818.6, 840.9, 850.2, 900.1, 900.7, 1042.5, 1042.8, 1267.6, 1267.8, 1267.9, 1268.0, 1271.4, 1272.0, 1272.5, 1273.3, 1275.1, 1275.1, 1275.3, 1275.5, 1626.4, 1626.9, 1627.3, 1650.9, 1651.0, 1651.2, 1688.1, 1688.3, 1689.0, 1689.8, 1690.8, 1706.4, 1707.1, 1707.8, 1708.2, 1714.9, 1715.0, 1717.5, 1718.7, 1719.0, 1722.5, 1722.7, 1728.2, 1729.0, 1729.9 ], "tpot_quantiles_ms": [ 131.3, 131.3, 131.3, 137.3, 137.3, 138.2, 139.4, 139.4, 139.4, 139.4, 140.1, 140.1, 142.6, 142.6, 142.7, 144.7, 144.7, 144.7, 146.4, 147.6, 147.6, 147.6, 147.7, 147.7, 147.7, 147.7, 148.6, 149.7, 149.7, 149.7, 149.7, 150.6, 150.6, 150.7, 150.7, 150.7, 150.7, 152.3, 152.3, 152.3, 152.3, 152.6, 156.1, 156.1, 156.1, 156.2, 157.1, 158.9, 161.9, 162.9, 167.7 ] } ] } } }