{
 "profile": "l4",
 "slo": {
  "ttft_ms": 1000.0,
  "tpot_ms": 100.0
 },
 "environment": {
  "sglang": "0.5.22",
  "sglang_kernel": "0.4.9",
  "flashinfer": "0.7.1",
  "torch": "2.14.1+cu130",
  "cuda": "13.0",
  "driver": "615.71.09\n615.71.09",
  "gpu": "NVIDIA L4",
  "compute_capability": "8.9",
  "gpu_memory_total_gib": 22.1,
  "gpu_memory_free_gib_at_start": 21.9,
  "python": "3.12.15",
  "machine": "x86_64",
  "profile": "l4",
  "models": [
   "Qwen/Qwen3-8B",
   "Qwen/Qwen3-8B-FP8",
   "Tengyunw/qwen3_8b_eagle3"
  ]
 },
 "load": {
  "vllm": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 13.51,
      "min": 13.47,
      "max": 13.53,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 191.84,
      "min": 191.43,
      "max": 198.04,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 704.56,
      "min": 694.91,
      "max": 705.01,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 60.36,
      "min": 60.35,
      "max": 60.36,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 104.16,
      "min": 104.15,
      "max": 104.17,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 104.54,
      "min": 104.4,
      "max": 104.71,
      "n": 3
     },
     "median_itl_ms": {
      "median": 60.47,
      "min": 60.46,
      "max": 60.52,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.09,
      "min": 0.09,
      "max": 0.09,
      "n": 3
     },
     "request_throughput": {
      "median": 0.11,
      "min": 0.11,
      "max": 0.11,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1290.0,
       "min": 1260.0,
       "max": 1290.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.48
     },
     "ttft_quantiles_ms": [
      75.9,
      75.9,
      76.7,
      78.1,
      174.3,
      175.3,
      178.3,
      178.3,
      180.3,
      180.4,
      183.0,
      183.1,
      183.4,
      185.6,
      185.8,
      186.3,
      186.7,
      186.7,
      187.0,
      187.8,
      188.3,
      188.4,
      189.2,
      191.1,
      191.3,
      191.7,
      192.4,
      192.6,
      198.3,
      203.5,
      221.7,
      223.9,
      315.2,
      321.4,
      321.4,
      372.9,
      386.6,
      389.8,
      429.3,
      503.9,
      512.1,
      513.5,
      530.4,
      538.2,
      624.5,
      702.8,
      703.6,
      703.7,
      704.7,
      705.3,
      707.3
     ],
     "tpot_quantiles_ms": [
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.4,
      60.4,
      60.4,
      60.4,
      60.7,
      60.7,
      62.3,
      66.4,
      71.4,
      71.4,
      72.0,
      72.3,
      86.7,
      87.3,
      88.4,
      92.0,
      93.7,
      95.0,
      98.0,
      104.1,
      104.1,
      104.1,
      104.2,
      104.2,
      104.2,
      104.2
     ]
    },
    {
     "x": 2.0,
     "output_throughput": {
      "median": 31.26,
      "min": 31.25,
      "max": 31.26,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 132.1,
      "min": 132.01,
      "max": 132.11,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 147.73,
      "min": 143.31,
      "max": 148.19,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 63.43,
      "min": 63.43,
      "max": 63.44,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 63.55,
      "min": 63.53,
      "max": 63.56,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 64.3,
      "min": 64.26,
      "max": 64.45,
      "n": 3
     },
     "median_itl_ms": {
      "median": 63.42,
      "min": 63.42,
      "max": 63.43,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "request_throughput": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1500.0,
       "min": 1500.0,
       "max": 2040.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.083
     },
     "ttft_quantiles_ms": [
      79.1,
      79.1,
      79.7,
      80.0,
      128.2,
      128.5,
      128.6,
      128.6,
      128.6,
      128.7,
      128.8,
      128.8,
      128.8,
      128.9,
      128.9,
      128.9,
      128.9,
      128.9,
      129.0,
      129.3,
      129.3,
      129.3,
      129.4,
      129.6,
      129.7,
      129.8,
      134.4,
      134.4,
      134.5,
      134.5,
      134.6,
      135.0,
      135.1,
      135.5,
      135.5,
      137.9,
      139.2,
      139.3,
      139.4,
      139.4,
      139.4,
      139.4,
      139.5,
      139.6,
      139.7,
      139.8,
      140.2,
      140.6,
      144.0,
      149.0,
      149.7
     ],
     "tpot_quantiles_ms": [
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.6,
      63.6
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 61.8,
      "min": 61.79,
      "max": 61.8,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 140.91,
      "min": 140.77,
      "max": 141.61,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 154.31,
      "min": 153.14,
      "max": 154.68,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 64.1,
      "min": 64.09,
      "max": 64.12,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 64.19,
      "min": 64.18,
      "max": 64.2,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 64.97,
      "min": 64.88,
      "max": 65.19,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.08,
      "min": 64.08,
      "max": 64.11,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "request_throughput": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1515.0,
       "min": 1515.0,
       "max": 1522.5,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      81.5,
      81.5,
      82.9,
      83.2,
      132.1,
      132.3,
      132.8,
      132.8,
      132.9,
      132.9,
      133.2,
      133.3,
      133.3,
      139.9,
      140.2,
      140.3,
      140.4,
      140.4,
      140.6,
      140.7,
      140.7,
      140.8,
      140.8,
      140.8,
      140.9,
      141.0,
      141.0,
      141.1,
      141.2,
      141.3,
      141.4,
      141.4,
      141.5,
      141.6,
      141.6,
      141.7,
      141.9,
      141.9,
      142.0,
      142.5,
      143.0,
      143.6,
      152.3,
      152.8,
      153.2,
      153.4,
      153.8,
      154.0,
      154.3,
      154.4,
      154.7
     ],
     "tpot_quantiles_ms": [
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2
     ]
    },
    {
     "x": 8.0,
     "output_throughput": {
      "median": 112.79,
      "min": 112.77,
      "max": 112.86,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 252.91,
      "min": 220.96,
      "max": 259.64,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1318.5,
      "min": 1313.64,
      "max": 1337.93,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 65.75,
      "min": 65.62,
      "max": 65.76,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 73.43,
      "min": 73.27,
      "max": 73.83,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 67.62,
      "min": 67.58,
      "max": 70.34,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.7,
      "min": 65.62,
      "max": 65.74,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.72,
      "min": 0.66,
      "max": 0.72,
      "n": 3
     },
     "request_throughput": {
      "median": 0.88,
      "min": 0.88,
      "max": 0.88,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1470.0,
       "min": 1470.0,
       "max": 1485.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      86.5,
      86.6,
      133.4,
      134.8,
      138.0,
      139.9,
      144.4,
      145.2,
      146.6,
      148.6,
      162.5,
      162.5,
      162.8,
      163.1,
      163.4,
      163.7,
      164.2,
      165.7,
      166.2,
      166.8,
      167.5,
      210.1,
      210.7,
      211.4,
      212.8,
      214.1,
      294.8,
      309.4,
      320.6,
      469.0,
      715.7,
      716.5,
      806.3,
      835.3,
      836.1,
      836.7,
      837.8,
      868.4,
      869.3,
      975.8,
      1117.2,
      1117.6,
      1200.7,
      1201.5,
      1257.4,
      1306.2,
      1307.2,
      1312.6,
      1313.9,
      1337.2,
      1338.1
     ],
     "tpot_quantiles_ms": [
      65.4,
      65.4,
      65.4,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.6,
      65.6,
      65.6,
      65.6,
      65.6,
      65.6,
      65.7,
      65.7,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      66.1,
      66.1,
      66.9,
      68.0,
      68.1,
      68.8,
      68.8,
      68.8,
      68.8,
      69.4,
      70.0,
      70.0,
      70.0,
      70.0,
      72.2,
      73.2,
      73.3,
      74.2
     ]
    },
    {
     "x": 16.0,
     "output_throughput": {
      "median": 200.9,
      "min": 200.51,
      "max": 201.21,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 288.72,
      "min": 281.85,
      "max": 293.73,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2515.04,
      "min": 2489.52,
      "max": 2543.37,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 69.5,
      "min": 69.44,
      "max": 69.92,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 85.85,
      "min": 85.54,
      "max": 85.96,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 80.14,
      "min": 79.5,
      "max": 104.71,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.47,
      "min": 69.41,
      "max": 69.64,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.01,
      "min": 1.0,
      "max": 1.03,
      "n": 3
     },
     "request_throughput": {
      "median": 1.57,
      "min": 1.57,
      "max": 1.57,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1425.0,
       "min": 1425.0,
       "max": 1440.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      96.4,
      173.8,
      175.1,
      180.1,
      181.5,
      185.4,
      186.2,
      187.7,
      202.1,
      205.6,
      211.0,
      215.3,
      216.2,
      217.9,
      222.5,
      231.5,
      239.4,
      240.6,
      241.5,
      243.7,
      252.0,
      253.3,
      263.9,
      270.1,
      271.2,
      272.4,
      352.7,
      839.9,
      854.6,
      856.9,
      880.8,
      883.0,
      900.5,
      1308.9,
      1310.3,
      1374.2,
      1437.4,
      1466.4,
      1693.3,
      1695.5,
      1697.8,
      1701.3,
      1738.6,
      1775.4,
      1998.3,
      2000.8,
      2033.8,
      2300.8,
      2489.9,
      2515.5,
      2543.6
     ],
     "tpot_quantiles_ms": [
      69.2,
      69.2,
      69.2,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.5,
      69.5,
      69.5,
      69.5,
      69.5,
      69.5,
      69.6,
      69.6,
      69.6,
      69.7,
      69.7,
      69.9,
      69.9,
      69.9,
      73.7,
      73.8,
      74.2,
      74.2,
      74.3,
      74.3,
      74.9,
      75.0,
      75.1,
      77.3,
      77.3,
      77.5,
      78.1,
      78.4,
      81.3,
      81.5,
      81.5,
      81.6,
      81.7,
      81.8,
      85.8,
      86.3
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 273.93,
      "min": 273.81,
      "max": 274.38,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 1813.49,
      "min": 1775.41,
      "max": 1833.2,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 4839.48,
      "min": 4822.54,
      "max": 4886.34,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 102.18,
      "min": 102.12,
      "max": 102.26,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 113.77,
      "min": 113.46,
      "max": 114.39,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 618.75,
      "min": 612.12,
      "max": 622.46,
      "n": 3
     },
     "median_itl_ms": {
      "median": 80.59,
      "min": 80.57,
      "max": 80.62,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.14,
      "min": 2.14,
      "max": 2.14,
      "n": 3
     },
     "completed": [
      128,
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1275.0,
       "min": 1260.0,
       "max": 1290.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      120.6,
      394.1,
      677.5,
      815.2,
      833.2,
      839.5,
      859.2,
      1275.9,
      1281.8,
      1284.4,
      1343.9,
      1394.7,
      1537.0,
      1612.5,
      1635.9,
      1669.4,
      1716.8,
      1733.9,
      1737.9,
      1768.1,
      1772.8,
      1776.9,
      1785.8,
      1799.9,
      1811.9,
      1823.1,
      1825.5,
      1829.2,
      1832.6,
      1836.2,
      1867.3,
      1869.8,
      1878.9,
      1883.5,
      1888.6,
      1906.2,
      1957.7,
      2167.9,
      2331.4,
      2381.3,
      2426.8,
      2443.1,
      2480.1,
      2499.5,
      3076.5,
      3099.2,
      3691.9,
      4292.2,
      4307.7,
      4823.1,
      4886.8
     ],
     "tpot_quantiles_ms": [
      80.2,
      80.3,
      82.7,
      87.1,
      87.3,
      91.4,
      95.1,
      96.2,
      100.1,
      100.8,
      100.9,
      100.9,
      100.9,
      101.1,
      101.4,
      101.5,
      101.6,
      101.6,
      101.7,
      101.7,
      101.7,
      101.8,
      101.8,
      101.9,
      102.1,
      102.2,
      102.3,
      102.3,
      102.4,
      102.4,
      102.5,
      102.7,
      103.2,
      103.4,
      103.6,
      104.0,
      104.2,
      104.5,
      104.6,
      105.0,
      105.2,
      105.8,
      107.1,
      107.9,
      108.7,
      108.8,
      110.0,
      111.3,
      111.6,
      112.9,
      115.8
     ]
    },
    {
     "x": 64.0,
     "output_throughput": {
      "median": 241.39,
      "min": 241.37,
      "max": 246.4,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 10953.28,
      "min": 10725.3,
      "max": 11528.47,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 26231.01,
      "min": 24116.5,
      "max": 27502.42,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 129.19,
      "min": 129.09,
      "max": 129.78,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 257.79,
      "min": 253.99,
      "max": 260.18,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1362.94,
      "min": 1204.34,
      "max": 1436.32,
      "n": 3
     },
     "median_itl_ms": {
      "median": 83.78,
      "min": 83.63,
      "max": 83.89,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 1.89,
      "min": 1.89,
      "max": 1.92,
      "n": 3
     },
     "completed": [
      256,
      256,
      256
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1012.5,
       "min": 990.0,
       "max": 1020.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.449
     },
     "ttft_quantiles_ms": [
      166.2,
      1320.5,
      1897.0,
      2841.8,
      3456.7,
      4257.5,
      4851.7,
      4973.4,
      5056.3,
      5176.4,
      5341.2,
      5461.7,
      5483.8,
      5556.8,
      5597.9,
      5727.9,
      5782.2,
      5866.8,
      6084.4,
      6660.5,
      7872.0,
      8939.8,
      9454.5,
      9916.7,
      10334.3,
      10787.4,
      11688.9,
      12075.6,
      12370.4,
      13849.1,
      15848.0,
      16068.4,
      16461.1,
      16575.8,
      16640.2,
      16695.1,
      16795.7,
      16883.4,
      18025.0,
      18633.2,
      19005.0,
      19271.2,
      19703.1,
      20236.6,
      20916.2,
      22655.7,
      23398.2,
      23808.4,
      24391.4,
      25051.1,
      27526.0
     ],
     "tpot_quantiles_ms": [
      71.9,
      80.6,
      119.8,
      119.9,
      120.5,
      120.7,
      120.7,
      121.1,
      121.3,
      121.4,
      121.5,
      121.6,
      121.7,
      121.8,
      122.0,
      122.1,
      122.2,
      122.5,
      122.7,
      122.7,
      123.0,
      123.8,
      124.7,
      125.6,
      127.7,
      129.2,
      132.3,
      138.6,
      141.8,
      144.5,
      146.6,
      150.8,
      154.1,
      159.4,
      162.6,
      164.7,
      165.8,
      168.6,
      170.7,
      171.7,
      173.1,
      174.1,
      176.4,
      180.2,
      183.7,
      186.4,
      191.2,
      195.1,
      204.7,
      241.0,
      305.8
     ]
    }
   ]
  },
  "sglang": {
   "kv_pool_tokens": 24094,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 13.84,
      "min": 13.65,
      "max": 14.07,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 184.79,
      "min": 181.36,
      "max": 186.46,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 621.85,
      "min": 565.56,
      "max": 662.01,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 60.16,
      "min": 60.15,
      "max": 60.16,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 101.76,
      "min": 101.64,
      "max": 103.31,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 103.53,
      "min": 103.5,
      "max": 103.83,
      "n": 3
     },
     "median_itl_ms": {
      "median": 60.2,
      "min": 60.2,
      "max": 60.21,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.1,
      "min": 0.08,
      "max": 0.1,
      "n": 3
     },
     "request_throughput": {
      "median": 0.11,
      "min": 0.11,
      "max": 0.11,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1290.0,
       "min": 1275.0,
       "max": 1290.0,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.434
     },
     "ttft_quantiles_ms": [
      71.0,
      71.0,
      71.5,
      71.9,
      170.0,
      170.4,
      173.9,
      174.1,
      176.0,
      178.1,
      178.2,
      178.5,
      179.5,
      179.7,
      179.7,
      180.3,
      180.6,
      180.6,
      180.7,
      180.9,
      180.9,
      181.3,
      181.4,
      182.2,
      183.6,
      183.6,
      183.8,
      184.7,
      186.0,
      186.7,
      188.3,
      209.7,
      292.1,
      303.5,
      303.5,
      308.7,
      374.1,
      390.6,
      454.1,
      469.8,
      470.1,
      476.7,
      499.4,
      515.9,
      523.3,
      534.9,
      543.1,
      560.7,
      566.4,
      639.2,
      694.7
     ],
     "tpot_quantiles_ms": [
      60.0,
      60.0,
      60.0,
      60.0,
      60.0,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.1,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.5,
      60.6,
      60.6,
      60.6,
      69.9,
      83.4,
      84.0,
      84.8,
      90.5,
      90.7,
      90.9,
      91.7,
      92.6,
      103.1,
      103.3,
      103.3,
      103.3,
      103.4,
      103.4
     ]
    },
    {
     "x": 2.0,
     "output_throughput": {
      "median": 31.2,
      "min": 31.2,
      "max": 31.2,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 115.74,
      "min": 115.34,
      "max": 117.34,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 135.16,
      "min": 134.03,
      "max": 136.48,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 63.65,
      "min": 63.65,
      "max": 63.66,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 63.87,
      "min": 63.86,
      "max": 63.87,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 65.08,
      "min": 64.92,
      "max": 65.27,
      "n": 3
     },
     "median_itl_ms": {
      "median": 63.52,
      "min": 63.5,
      "max": 63.52,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "request_throughput": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1500.0,
       "min": 1500.0,
       "max": 1515.0,
       "n": 3
      },
      "max_temp_c": 86.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      96.3,
      96.3,
      97.0,
      97.0,
      97.0,
      97.0,
      97.3,
      97.3,
      97.9,
      98.0,
      98.0,
      98.0,
      98.1,
      98.1,
      98.1,
      98.2,
      98.6,
      98.6,
      98.6,
      98.8,
      98.8,
      99.0,
      99.1,
      99.3,
      99.3,
      102.7,
      131.6,
      131.8,
      132.0,
      132.2,
      132.2,
      132.3,
      132.5,
      132.7,
      132.7,
      132.8,
      132.9,
      133.0,
      133.0,
      133.2,
      133.3,
      133.3,
      133.4,
      133.6,
      133.7,
      133.7,
      134.0,
      134.0,
      134.6,
      135.4,
      136.8
     ],
     "tpot_quantiles_ms": [
      63.4,
      63.4,
      63.4,
      63.4,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.6,
      63.6,
      63.6,
      63.6,
      63.6,
      63.6,
      63.6,
      63.6,
      63.7,
      63.7,
      63.7,
      63.7,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.8,
      63.9,
      63.9,
      63.9,
      63.9,
      63.9
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 61.55,
      "min": 61.53,
      "max": 61.56,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 137.27,
      "min": 137.27,
      "max": 138.38,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 138.42,
      "min": 138.31,
      "max": 140.63,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 64.36,
      "min": 64.33,
      "max": 64.37,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 64.78,
      "min": 64.75,
      "max": 64.79,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 67.06,
      "min": 64.97,
      "max": 67.1,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.33,
      "min": 64.3,
      "max": 64.33,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "request_throughput": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1530.0,
       "min": 1515.0,
       "max": 1575.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      86.2,
      86.2,
      86.5,
      86.5,
      86.7,
      86.8,
      86.9,
      87.2,
      87.2,
      87.8,
      87.9,
      88.1,
      89.3,
      136.7,
      136.7,
      136.7,
      137.0,
      137.0,
      137.1,
      137.1,
      137.2,
      137.2,
      137.3,
      137.4,
      137.6,
      137.7,
      137.7,
      137.7,
      137.7,
      137.7,
      137.7,
      137.8,
      137.9,
      137.9,
      137.9,
      137.9,
      138.0,
      138.1,
      138.2,
      138.2,
      138.3,
      138.3,
      138.4,
      138.5,
      138.6,
      138.6,
      139.0,
      139.2,
      139.3,
      140.1,
      140.7
     ],
     "tpot_quantiles_ms": [
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8
     ]
    },
    {
     "x": 8.0,
     "output_throughput": {
      "median": 112.46,
      "min": 112.38,
      "max": 112.58,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 169.98,
      "min": 160.27,
      "max": 200.89,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1277.35,
      "min": 1230.77,
      "max": 1365.42,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 66.09,
      "min": 66.01,
      "max": 66.32,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 74.63,
      "min": 74.26,
      "max": 74.74,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 70.77,
      "min": 70.35,
      "max": 73.3,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.99,
      "min": 65.93,
      "max": 66.04,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.63,
      "min": 0.6,
      "max": 0.66,
      "n": 3
     },
     "request_throughput": {
      "median": 0.88,
      "min": 0.88,
      "max": 0.88,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1470.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      86.3,
      88.0,
      89.0,
      101.4,
      137.6,
      138.6,
      139.0,
      139.3,
      139.7,
      140.0,
      140.6,
      141.0,
      141.6,
      142.3,
      143.0,
      153.9,
      154.4,
      154.7,
      155.0,
      155.4,
      155.8,
      162.5,
      163.4,
      164.4,
      173.8,
      184.3,
      196.3,
      203.9,
      219.4,
      488.6,
      504.3,
      504.3,
      529.9,
      624.2,
      625.0,
      803.2,
      805.3,
      1205.7,
      1206.3,
      1211.3,
      1212.3,
      1212.9,
      1217.0,
      1218.0,
      1219.3,
      1230.6,
      1276.4,
      1277.1,
      1363.8,
      1364.7,
      1365.6
     ],
     "tpot_quantiles_ms": [
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.0,
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.3,
      66.3,
      66.3,
      66.3,
      66.3,
      66.3,
      66.4,
      66.4,
      66.4,
      66.5,
      66.9,
      69.5,
      69.5,
      71.4,
      71.4,
      71.6,
      71.7,
      71.7,
      72.6,
      74.2,
      74.3,
      75.0
     ]
    },
    {
     "x": 16.0,
     "output_throughput": {
      "median": 201.13,
      "min": 201.09,
      "max": 201.21,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 211.91,
      "min": 211.27,
      "max": 213.69,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2425.75,
      "min": 2412.93,
      "max": 2466.53,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 70.06,
      "min": 70.02,
      "max": 70.37,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 87.28,
      "min": 87.14,
      "max": 87.31,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 81.41,
      "min": 80.67,
      "max": 82.32,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.64,
      "min": 69.59,
      "max": 69.77,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.96,
      "min": 0.96,
      "max": 0.98,
      "n": 3
     },
     "request_throughput": {
      "median": 1.57,
      "min": 1.57,
      "max": 1.57,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1470.0,
       "min": 1440.0,
       "max": 1470.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      89.8,
      92.7,
      150.2,
      151.9,
      152.9,
      154.1,
      155.4,
      157.6,
      159.0,
      159.5,
      159.9,
      160.3,
      160.9,
      188.3,
      194.5,
      201.3,
      201.9,
      202.6,
      203.4,
      204.0,
      206.0,
      210.1,
      210.7,
      211.4,
      211.7,
      212.4,
      213.5,
      490.1,
      612.3,
      621.3,
      624.4,
      1653.8,
      1655.7,
      1678.4,
      1679.9,
      1777.2,
      1778.7,
      1788.0,
      1790.4,
      1791.6,
      1809.5,
      1810.7,
      1818.7,
      1820.6,
      2354.6,
      2384.4,
      2398.9,
      2412.1,
      2425.4,
      2464.7,
      2466.7
     ],
     "tpot_quantiles_ms": [
      69.6,
      69.6,
      69.6,
      69.6,
      69.7,
      69.7,
      69.7,
      69.7,
      69.7,
      69.7,
      69.7,
      69.7,
      69.8,
      69.8,
      69.8,
      69.8,
      69.8,
      70.0,
      70.0,
      70.0,
      70.0,
      70.0,
      70.0,
      70.1,
      70.1,
      70.1,
      70.1,
      70.1,
      70.1,
      70.4,
      70.4,
      70.6,
      70.9,
      74.2,
      74.3,
      74.3,
      74.3,
      74.4,
      74.4,
      75.1,
      75.1,
      75.6,
      75.6,
      76.2,
      76.2,
      83.5,
      83.6,
      84.2,
      85.0,
      87.0,
      87.8
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 271.28,
      "min": 270.26,
      "max": 272.19,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 2870.77,
      "min": 2823.43,
      "max": 2873.49,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 4782.15,
      "min": 4776.84,
      "max": 4893.23,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 96.79,
      "min": 96.48,
      "max": 96.82,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 117.84,
      "min": 117.6,
      "max": 117.85,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 102.06,
      "min": 101.77,
      "max": 102.57,
      "n": 3
     },
     "median_itl_ms": {
      "median": 81.57,
      "min": 81.53,
      "max": 81.62,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.12,
      "min": 2.11,
      "max": 2.13,
      "n": 3
     },
     "completed": [
      128,
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1320.0,
       "min": 1320.0,
       "max": 1320.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      106.1,
      199.8,
      495.4,
      508.7,
      625.9,
      1094.7,
      1098.1,
      1447.7,
      1648.3,
      1652.0,
      1664.8,
      1679.7,
      1681.9,
      1685.1,
      1688.7,
      1778.3,
      1811.0,
      2260.2,
      2261.4,
      2643.4,
      2801.4,
      2806.8,
      2822.9,
      2861.6,
      2865.3,
      2867.5,
      2870.4,
      2873.6,
      2942.2,
      3005.6,
      3408.4,
      3419.6,
      3862.9,
      3967.3,
      3971.3,
      3993.7,
      4048.5,
      4051.0,
      4057.1,
      4062.4,
      4107.4,
      4195.1,
      4198.3,
      4488.6,
      4674.4,
      4759.0,
      4770.1,
      4772.0,
      4780.5,
      4839.0,
      4894.5
     ],
     "tpot_quantiles_ms": [
      81.6,
      81.6,
      81.7,
      81.8,
      81.8,
      81.8,
      81.9,
      81.9,
      82.6,
      82.8,
      86.3,
      87.0,
      87.3,
      87.4,
      87.5,
      87.5,
      88.1,
      88.5,
      88.5,
      90.1,
      91.8,
      91.8,
      92.1,
      95.7,
      96.2,
      96.5,
      96.8,
      96.8,
      96.9,
      97.3,
      97.9,
      99.1,
      99.2,
      100.9,
      101.1,
      105.1,
      105.3,
      105.6,
      106.2,
      106.2,
      106.2,
      106.4,
      107.2,
      108.2,
      108.6,
      110.0,
      110.2,
      114.5,
      115.5,
      117.8,
      119.0
     ]
    },
    {
     "x": 64.0,
     "output_throughput": {
      "median": 256.62,
      "min": 255.76,
      "max": 260.48,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 18355.96,
      "min": 18291.44,
      "max": 18393.38,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 31321.78,
      "min": 30933.0,
      "max": 32221.24,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 106.17,
      "min": 106.09,
      "max": 107.03,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 159.34,
      "min": 152.5,
      "max": 161.07,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 114.31,
      "min": 114.09,
      "max": 117.69,
      "n": 3
     },
     "median_itl_ms": {
      "median": 83.82,
      "min": 83.73,
      "max": 83.85,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.0,
      "min": 2.0,
      "max": 2.03,
      "n": 3
     },
     "completed": [
      256,
      256,
      256
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1215.0,
       "min": 1200.0,
       "max": 1215.0,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.344
     },
     "ttft_quantiles_ms": [
      197.1,
      1836.1,
      1841.4,
      3006.2,
      3025.0,
      4177.3,
      4667.3,
      4767.1,
      4775.4,
      5345.6,
      5437.3,
      5476.4,
      5497.5,
      5597.4,
      6520.7,
      7594.2,
      10300.4,
      10667.0,
      11618.3,
      17192.1,
      17275.0,
      17299.1,
      17309.8,
      17435.6,
      17506.6,
      18292.9,
      18394.6,
      18463.2,
      18493.6,
      18613.5,
      18995.3,
      19481.2,
      19544.7,
      19652.7,
      19693.2,
      19796.3,
      20548.1,
      20669.5,
      20691.1,
      20856.5,
      20918.4,
      22631.3,
      23200.6,
      24401.3,
      25815.9,
      25926.0,
      27066.7,
      28080.6,
      29734.2,
      31075.8,
      33619.1
     ],
     "tpot_quantiles_ms": [
      83.7,
      83.9,
      83.9,
      84.0,
      84.1,
      84.3,
      84.7,
      85.2,
      89.6,
      89.7,
      90.3,
      90.3,
      90.7,
      92.7,
      93.4,
      93.9,
      96.9,
      96.9,
      99.0,
      99.1,
      99.1,
      99.5,
      100.0,
      101.9,
      103.1,
      106.2,
      107.1,
      108.4,
      108.5,
      108.6,
      109.3,
      111.1,
      111.8,
      112.9,
      115.4,
      117.7,
      117.7,
      117.8,
      118.2,
      118.7,
      119.6,
      120.2,
      124.0,
      130.5,
      132.2,
      133.0,
      138.8,
      148.2,
      152.5,
      159.3,
      161.1
     ]
    }
   ]
  },
  "vllm-b": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 13.49,
      "min": 13.09,
      "max": 13.79,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 191.28,
      "min": 190.87,
      "max": 195.99,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 695.78,
      "min": 687.07,
      "max": 706.25,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 60.36,
      "min": 60.34,
      "max": 60.36,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 104.17,
      "min": 104.14,
      "max": 104.17,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 104.53,
      "min": 104.4,
      "max": 104.56,
      "n": 3
     },
     "median_itl_ms": {
      "median": 60.52,
      "min": 60.41,
      "max": 60.56,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.09,
      "min": 0.07,
      "max": 0.09,
      "n": 3
     },
     "request_throughput": {
      "median": 0.11,
      "min": 0.1,
      "max": 0.11,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1275.0,
       "min": 1215.0,
       "max": 1290.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.483
     },
     "ttft_quantiles_ms": [
      76.1,
      76.1,
      77.3,
      78.0,
      168.1,
      173.2,
      173.6,
      180.9,
      181.1,
      182.5,
      182.6,
      183.0,
      183.7,
      184.0,
      186.0,
      186.1,
      186.2,
      186.2,
      186.5,
      187.4,
      187.7,
      188.4,
      189.1,
      189.5,
      190.3,
      190.9,
      191.5,
      191.7,
      193.1,
      202.9,
      205.5,
      217.8,
      237.0,
      315.8,
      315.8,
      405.9,
      439.7,
      445.6,
      452.8,
      496.4,
      508.9,
      563.0,
      571.3,
      575.1,
      578.1,
      641.4,
      704.3,
      705.4,
      705.4,
      706.3,
      706.4
     ],
     "tpot_quantiles_ms": [
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.4,
      60.4,
      60.4,
      60.4,
      60.6,
      60.9,
      63.0,
      68.3,
      68.3,
      74.8,
      75.3,
      85.6,
      85.7,
      87.0,
      87.8,
      103.7,
      103.8,
      104.1,
      104.1,
      104.1,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2
     ]
    },
    {
     "x": 2.0,
     "output_throughput": {
      "median": 31.26,
      "min": 31.26,
      "max": 31.26,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 132.2,
      "min": 132.08,
      "max": 132.55,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 147.43,
      "min": 142.43,
      "max": 149.08,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 63.43,
      "min": 63.42,
      "max": 63.43,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 63.54,
      "min": 63.53,
      "max": 63.55,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 64.27,
      "min": 64.27,
      "max": 64.29,
      "n": 3
     },
     "median_itl_ms": {
      "median": 63.41,
      "min": 63.41,
      "max": 63.42,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "request_throughput": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1500.0,
       "min": 1500.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      78.0,
      78.0,
      80.0,
      80.2,
      128.3,
      128.5,
      128.6,
      128.7,
      128.8,
      129.0,
      129.0,
      129.0,
      129.1,
      129.2,
      129.2,
      129.3,
      129.3,
      129.3,
      129.3,
      129.3,
      129.3,
      129.5,
      129.6,
      129.6,
      130.4,
      130.6,
      133.6,
      134.6,
      134.7,
      134.8,
      134.9,
      135.0,
      135.0,
      135.0,
      135.0,
      139.1,
      139.3,
      139.5,
      139.6,
      139.7,
      139.7,
      140.0,
      140.0,
      140.2,
      140.3,
      141.0,
      141.1,
      141.2,
      142.7,
      148.6,
      150.5
     ],
     "tpot_quantiles_ms": [
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 61.79,
      "min": 61.79,
      "max": 61.8,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 141.61,
      "min": 139.85,
      "max": 142.08,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 154.73,
      "min": 153.3,
      "max": 155.71,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 64.1,
      "min": 64.09,
      "max": 64.1,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 64.2,
      "min": 64.19,
      "max": 64.2,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 65.17,
      "min": 64.84,
      "max": 65.81,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.1,
      "min": 64.09,
      "max": 64.1,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "request_throughput": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1560.0,
       "min": 1500.0,
       "max": 1560.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      81.6,
      81.6,
      81.8,
      84.1,
      131.2,
      131.9,
      132.1,
      132.3,
      132.5,
      132.7,
      133.2,
      133.5,
      134.2,
      139.0,
      139.1,
      139.6,
      139.6,
      139.6,
      140.1,
      140.1,
      140.1,
      140.2,
      140.6,
      140.6,
      140.7,
      141.0,
      141.2,
      141.2,
      141.3,
      141.6,
      141.6,
      141.8,
      142.1,
      142.1,
      142.1,
      142.3,
      142.5,
      142.9,
      143.0,
      143.4,
      143.9,
      144.4,
      152.4,
      153.0,
      153.4,
      153.9,
      154.5,
      154.8,
      154.8,
      155.3,
      155.8
     ],
     "tpot_quantiles_ms": [
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2
     ]
    },
    {
     "x": 8.0,
     "output_throughput": {
      "median": 112.79,
      "min": 112.67,
      "max": 112.82,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 252.88,
      "min": 251.67,
      "max": 258.62,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1320.06,
      "min": 1311.34,
      "max": 1331.49,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 65.75,
      "min": 65.64,
      "max": 65.76,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 73.31,
      "min": 73.28,
      "max": 73.72,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 67.12,
      "min": 66.81,
      "max": 67.81,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.69,
      "min": 65.66,
      "max": 65.76,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.66,
      "min": 0.66,
      "max": 0.69,
      "n": 3
     },
     "request_throughput": {
      "median": 0.88,
      "min": 0.88,
      "max": 0.88,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1455.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      86.0,
      87.0,
      134.1,
      134.7,
      138.2,
      139.1,
      139.7,
      140.8,
      161.1,
      161.6,
      162.2,
      162.6,
      162.9,
      163.2,
      163.5,
      163.8,
      164.2,
      164.5,
      164.8,
      211.3,
      211.7,
      211.9,
      212.2,
      212.8,
      213.4,
      214.4,
      291.4,
      310.5,
      335.0,
      583.6,
      592.3,
      699.3,
      700.5,
      702.4,
      703.4,
      704.6,
      855.0,
      982.9,
      984.0,
      1101.4,
      1113.8,
      1114.4,
      1217.9,
      1218.8,
      1225.8,
      1226.4,
      1227.2,
      1286.3,
      1292.4,
      1311.5,
      1361.8
     ],
     "tpot_quantiles_ms": [
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.6,
      65.6,
      65.6,
      65.6,
      65.6,
      65.6,
      65.6,
      65.6,
      65.7,
      65.7,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      66.0,
      66.6,
      67.0,
      67.0,
      67.7,
      67.9,
      69.3,
      69.3,
      69.9,
      69.9,
      69.9,
      70.8,
      70.8,
      71.0,
      71.1,
      73.0,
      73.4,
      74.0
     ]
    },
    {
     "x": 16.0,
     "output_throughput": {
      "median": 201.3,
      "min": 200.87,
      "max": 201.33,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 288.25,
      "min": 280.31,
      "max": 289.93,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2487.38,
      "min": 2485.68,
      "max": 2500.83,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 69.49,
      "min": 69.45,
      "max": 69.68,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 85.6,
      "min": 85.47,
      "max": 85.77,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 84.81,
      "min": 80.64,
      "max": 88.74,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.5,
      "min": 69.4,
      "max": 69.53,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.01,
      "min": 1.01,
      "max": 1.03,
      "n": 3
     },
     "request_throughput": {
      "median": 1.57,
      "min": 1.57,
      "max": 1.57,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1455.0,
       "min": 1425.0,
       "max": 1455.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      96.5,
      183.6,
      184.8,
      186.3,
      189.4,
      191.1,
      192.6,
      193.9,
      195.1,
      196.3,
      218.8,
      219.5,
      220.2,
      221.0,
      221.5,
      222.7,
      223.8,
      225.0,
      240.9,
      241.7,
      242.8,
      244.5,
      262.3,
      263.2,
      264.8,
      265.9,
      351.2,
      834.2,
      837.5,
      842.2,
      877.1,
      879.0,
      896.4,
      1305.1,
      1306.2,
      1371.2,
      1422.9,
      1445.1,
      1689.2,
      1691.3,
      1696.9,
      1698.4,
      1706.4,
      1774.3,
      1984.1,
      2013.3,
      2027.4,
      2296.2,
      2486.2,
      2499.7,
      2519.1
     ],
     "tpot_quantiles_ms": [
      69.2,
      69.2,
      69.2,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.5,
      69.5,
      69.5,
      69.5,
      69.5,
      69.5,
      69.6,
      69.6,
      69.8,
      69.9,
      73.7,
      73.8,
      74.2,
      74.2,
      74.2,
      74.2,
      74.9,
      74.9,
      75.0,
      77.2,
      77.3,
      77.3,
      77.9,
      78.4,
      81.2,
      81.3,
      81.3,
      81.4,
      81.5,
      81.7,
      85.5,
      86.2
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 275.14,
      "min": 274.84,
      "max": 275.15,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 1800.22,
      "min": 1763.26,
      "max": 1819.41,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 4821.25,
      "min": 4772.07,
      "max": 4847.92,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 101.79,
      "min": 101.73,
      "max": 102.2,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 113.44,
      "min": 113.3,
      "max": 113.73,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 613.31,
      "min": 608.38,
      "max": 613.67,
      "n": 3
     },
     "median_itl_ms": {
      "median": 80.53,
      "min": 80.5,
      "max": 80.59,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.15,
      "min": 2.15,
      "max": 2.15,
      "n": 3
     },
     "completed": [
      128,
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1275.0,
       "min": 1267.5,
       "max": 1275.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      123.5,
      393.7,
      527.1,
      676.7,
      823.4,
      836.1,
      934.0,
      1250.8,
      1276.2,
      1282.2,
      1386.2,
      1421.9,
      1454.4,
      1566.7,
      1611.8,
      1634.4,
      1667.9,
      1723.0,
      1729.0,
      1732.0,
      1737.0,
      1758.2,
      1772.1,
      1781.1,
      1796.7,
      1799.4,
      1802.0,
      1820.1,
      1827.6,
      1829.6,
      1831.4,
      1834.7,
      1838.0,
      1849.2,
      1863.2,
      1867.3,
      1955.9,
      2102.0,
      2284.9,
      2337.6,
      2396.2,
      2422.2,
      2441.5,
      2474.9,
      3011.1,
      3081.9,
      3595.3,
      3836.2,
      4304.6,
      4772.1,
      4848.2
     ],
     "tpot_quantiles_ms": [
      80.1,
      80.3,
      82.6,
      86.0,
      87.1,
      91.4,
      94.7,
      95.6,
      99.9,
      100.8,
      100.8,
      100.9,
      100.9,
      100.9,
      101.1,
      101.2,
      101.4,
      101.5,
      101.5,
      101.6,
      101.6,
      101.6,
      101.7,
      101.7,
      101.7,
      101.8,
      101.9,
      102.0,
      102.2,
      102.3,
      102.3,
      102.4,
      102.8,
      103.0,
      103.3,
      103.5,
      103.6,
      103.9,
      104.5,
      104.6,
      104.9,
      105.4,
      106.7,
      107.7,
      108.6,
      108.8,
      109.5,
      110.6,
      111.2,
      112.8,
      114.6
     ]
    },
    {
     "x": 64.0,
     "output_throughput": {
      "median": 242.32,
      "min": 242.04,
      "max": 243.77,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 11180.53,
      "min": 10245.98,
      "max": 11319.61,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 26201.73,
      "min": 25347.15,
      "max": 26522.34,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 129.15,
      "min": 128.08,
      "max": 129.19,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 261.51,
      "min": 259.88,
      "max": 263.02,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1232.94,
      "min": 1187.45,
      "max": 1436.99,
      "n": 3
     },
     "median_itl_ms": {
      "median": 83.75,
      "min": 83.63,
      "max": 83.76,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 1.89,
      "min": 1.89,
      "max": 1.9,
      "n": 3
     },
     "completed": [
      256,
      256,
      256
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1005.0,
       "min": 1005.0,
       "max": 1005.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.443
     },
     "ttft_quantiles_ms": [
      174.1,
      1316.4,
      1901.1,
      2538.6,
      3648.2,
      4278.4,
      4874.5,
      4974.8,
      5029.9,
      5114.1,
      5436.4,
      5512.6,
      5539.4,
      5589.8,
      5606.1,
      5718.8,
      5812.4,
      5898.5,
      6087.5,
      6674.0,
      7514.2,
      8720.9,
      9381.3,
      9872.1,
      10233.5,
      10668.4,
      11251.0,
      11539.0,
      12472.2,
      13780.9,
      15932.5,
      16049.5,
      16507.8,
      16552.6,
      16578.7,
      16662.8,
      16707.9,
      16905.7,
      17930.5,
      18484.4,
      18935.8,
      19254.1,
      19732.5,
      20322.9,
      20959.2,
      23309.6,
      24020.0,
      24108.9,
      24728.8,
      25410.5,
      26675.9
     ],
     "tpot_quantiles_ms": [
      71.8,
      79.3,
      119.9,
      120.3,
      120.4,
      120.5,
      120.6,
      120.7,
      120.9,
      121.0,
      121.2,
      121.3,
      121.4,
      121.5,
      121.6,
      122.0,
      122.2,
      122.2,
      122.3,
      122.6,
      122.7,
      123.6,
      124.6,
      125.3,
      126.7,
      128.4,
      131.2,
      136.0,
      140.2,
      144.3,
      148.8,
      150.8,
      155.3,
      161.0,
      164.0,
      167.3,
      170.2,
      171.1,
      173.9,
      175.0,
      176.9,
      178.7,
      180.4,
      181.6,
      184.3,
      187.4,
      191.3,
      193.4,
      206.2,
      242.9,
      304.5
     ]
    }
   ]
  },
  "vllm-eager": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 13.15,
      "min": 13.06,
      "max": 13.36,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 190.05,
      "min": 185.96,
      "max": 194.92,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 683.52,
      "min": 641.67,
      "max": 684.15,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 61.07,
      "min": 61.06,
      "max": 61.07,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 106.06,
      "min": 105.92,
      "max": 106.47,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 106.76,
      "min": 106.75,
      "max": 106.97,
      "n": 3
     },
     "median_itl_ms": {
      "median": 61.21,
      "min": 61.2,
      "max": 61.21,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.07,
      "min": 0.07,
      "max": 0.08,
      "n": 3
     },
     "request_throughput": {
      "median": 0.1,
      "min": 0.1,
      "max": 0.1,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1290.0,
       "min": 1260.0,
       "max": 1305.0,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.48
     },
     "ttft_quantiles_ms": [
      90.0,
      90.0,
      91.1,
      91.8,
      164.4,
      168.4,
      169.7,
      175.3,
      178.6,
      181.5,
      182.6,
      183.1,
      183.2,
      183.6,
      184.1,
      184.6,
      184.7,
      184.7,
      184.9,
      185.5,
      186.4,
      187.0,
      187.5,
      188.1,
      188.4,
      188.6,
      191.3,
      191.7,
      192.9,
      198.6,
      203.3,
      215.3,
      370.3,
      441.7,
      441.7,
      455.6,
      460.3,
      536.7,
      540.3,
      542.3,
      547.4,
      548.1,
      555.2,
      557.8,
      558.4,
      613.1,
      617.4,
      628.2,
      644.0,
      705.6,
      706.9
     ],
     "tpot_quantiles_ms": [
      60.7,
      60.7,
      60.8,
      60.8,
      60.9,
      60.9,
      60.9,
      60.9,
      60.9,
      60.9,
      60.9,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.0,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.6,
      61.7,
      61.8,
      64.5,
      65.3,
      65.3,
      65.4,
      69.0,
      100.3,
      100.6,
      101.0,
      101.5,
      101.9,
      101.9,
      102.7,
      103.7,
      106.4,
      106.4,
      106.5,
      106.5,
      106.5,
      106.5
     ]
    },
    {
     "x": 2.0,
     "output_throughput": {
      "median": 30.75,
      "min": 30.74,
      "max": 30.75,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 140.87,
      "min": 140.0,
      "max": 140.89,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 148.64,
      "min": 148.08,
      "max": 153.66,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 64.45,
      "min": 64.42,
      "max": 64.48,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 64.56,
      "min": 64.54,
      "max": 64.63,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 65.1,
      "min": 65.05,
      "max": 65.12,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.48,
      "min": 64.47,
      "max": 64.49,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "request_throughput": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1515.0,
       "min": 1515.0,
       "max": 1515.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      89.7,
      89.7,
      91.3,
      96.0,
      129.2,
      129.3,
      129.3,
      129.3,
      129.6,
      130.3,
      130.7,
      135.1,
      135.5,
      135.5,
      135.6,
      135.7,
      135.8,
      135.8,
      136.0,
      136.2,
      136.9,
      139.9,
      140.1,
      140.4,
      140.4,
      140.4,
      140.7,
      140.7,
      140.9,
      140.9,
      140.9,
      141.0,
      141.0,
      141.1,
      141.1,
      141.1,
      141.2,
      141.4,
      141.4,
      141.8,
      141.9,
      141.9,
      143.0,
      143.2,
      143.6,
      145.1,
      146.8,
      147.4,
      148.2,
      149.6,
      155.7
     ],
     "tpot_quantiles_ms": [
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.5,
      64.6,
      64.6,
      64.6,
      64.6
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 60.65,
      "min": 60.65,
      "max": 60.68,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 142.57,
      "min": 142.2,
      "max": 143.38,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 156.44,
      "min": 155.33,
      "max": 157.13,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 65.3,
      "min": 65.28,
      "max": 65.31,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 65.36,
      "min": 65.36,
      "max": 65.38,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 66.07,
      "min": 65.83,
      "max": 66.14,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.31,
      "min": 65.28,
      "max": 65.31,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.47,
      "min": 0.47,
      "max": 0.47,
      "n": 3
     },
     "request_throughput": {
      "median": 0.47,
      "min": 0.47,
      "max": 0.47,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1560.0,
       "min": 1515.0,
       "max": 1560.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      94.8,
      94.8,
      95.2,
      98.0,
      133.4,
      139.5,
      140.3,
      140.7,
      140.7,
      140.8,
      140.9,
      141.2,
      141.3,
      141.3,
      141.4,
      141.6,
      141.8,
      141.8,
      141.9,
      141.9,
      141.9,
      142.0,
      142.0,
      142.1,
      142.4,
      142.5,
      142.5,
      142.6,
      143.0,
      143.1,
      143.1,
      143.2,
      143.7,
      143.7,
      143.7,
      144.0,
      144.6,
      144.8,
      145.1,
      148.3,
      149.2,
      149.6,
      154.5,
      155.0,
      155.4,
      155.6,
      156.1,
      156.3,
      156.5,
      156.8,
      157.2
     ],
     "tpot_quantiles_ms": [
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.4,
      65.4,
      65.4,
      65.4,
      65.4,
      65.4,
      65.4
     ]
    },
    {
     "x": 8.0,
     "output_throughput": {
      "median": 111.22,
      "min": 111.16,
      "max": 111.36,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 205.93,
      "min": 202.95,
      "max": 215.19,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1261.21,
      "min": 1229.24,
      "max": 1293.53,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 66.98,
      "min": 66.96,
      "max": 66.99,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 74.23,
      "min": 74.22,
      "max": 74.66,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 68.19,
      "min": 68.06,
      "max": 68.27,
      "n": 3
     },
     "median_itl_ms": {
      "median": 67.0,
      "min": 66.99,
      "max": 67.02,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.65,
      "min": 0.65,
      "max": 0.71,
      "n": 3
     },
     "request_throughput": {
      "median": 0.87,
      "min": 0.87,
      "max": 0.87,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1515.0,
       "min": 1500.0,
       "max": 1515.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      102.7,
      103.2,
      143.0,
      143.6,
      144.2,
      144.5,
      145.0,
      145.1,
      145.6,
      146.3,
      146.8,
      147.1,
      147.6,
      148.7,
      149.8,
      163.6,
      164.1,
      164.8,
      165.2,
      165.8,
      166.4,
      166.4,
      166.8,
      167.1,
      167.5,
      168.2,
      244.7,
      295.4,
      304.8,
      670.5,
      686.0,
      686.4,
      746.5,
      748.2,
      755.1,
      756.7,
      777.9,
      811.1,
      812.4,
      1167.3,
      1168.4,
      1179.4,
      1180.6,
      1201.9,
      1202.8,
      1228.1,
      1229.4,
      1259.6,
      1260.7,
      1292.7,
      1293.7
     ],
     "tpot_quantiles_ms": [
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      66.9,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.0,
      67.2,
      69.8,
      69.8,
      70.2,
      70.2,
      70.3,
      70.3,
      71.0,
      71.0,
      71.0,
      71.1,
      71.1,
      73.9,
      74.3,
      75.0
     ]
    },
    {
     "x": 16.0,
     "output_throughput": {
      "median": 198.37,
      "min": 198.24,
      "max": 198.38,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 278.06,
      "min": 266.24,
      "max": 280.88,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2448.76,
      "min": 2399.23,
      "max": 2457.84,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 71.12,
      "min": 71.06,
      "max": 71.22,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 86.47,
      "min": 86.46,
      "max": 86.58,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 79.67,
      "min": 79.58,
      "max": 85.27,
      "n": 3
     },
     "median_itl_ms": {
      "median": 71.05,
      "min": 71.0,
      "max": 71.07,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.99,
      "min": 0.99,
      "max": 1.02,
      "n": 3
     },
     "request_throughput": {
      "median": 1.55,
      "min": 1.55,
      "max": 1.55,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1455.0,
       "min": 1425.0,
       "max": 1455.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      114.5,
      156.3,
      158.0,
      160.6,
      164.5,
      171.8,
      173.1,
      175.1,
      176.4,
      177.7,
      179.8,
      183.7,
      199.4,
      200.7,
      201.6,
      202.1,
      202.9,
      203.2,
      204.0,
      205.4,
      219.3,
      220.4,
      221.5,
      222.5,
      240.4,
      253.4,
      342.6,
      812.2,
      814.1,
      820.3,
      846.3,
      849.8,
      864.9,
      1262.7,
      1263.5,
      1330.6,
      1387.7,
      1391.4,
      1637.8,
      1640.0,
      1653.1,
      1654.6,
      1668.7,
      1740.6,
      1965.5,
      1967.5,
      1969.4,
      2245.1,
      2399.8,
      2448.4,
      2458.0
     ],
     "tpot_quantiles_ms": [
      70.7,
      70.7,
      70.8,
      70.8,
      70.8,
      70.8,
      70.8,
      70.8,
      70.9,
      70.9,
      70.9,
      70.9,
      70.9,
      70.9,
      71.0,
      71.0,
      71.0,
      71.0,
      71.1,
      71.1,
      71.1,
      71.1,
      71.1,
      71.1,
      71.1,
      71.2,
      71.2,
      71.2,
      71.3,
      75.3,
      75.3,
      75.4,
      75.4,
      75.4,
      75.4,
      76.1,
      76.1,
      76.2,
      78.3,
      78.4,
      78.4,
      79.4,
      79.5,
      82.3,
      82.3,
      82.3,
      82.4,
      82.6,
      82.7,
      86.4,
      87.2
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 273.22,
      "min": 272.36,
      "max": 273.34,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 1767.61,
      "min": 1711.08,
      "max": 1793.45,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 4655.34,
      "min": 4650.57,
      "max": 4694.45,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 103.11,
      "min": 102.88,
      "max": 103.35,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 114.32,
      "min": 114.28,
      "max": 114.9,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 600.08,
      "min": 595.26,
      "max": 600.88,
      "n": 3
     },
     "median_itl_ms": {
      "median": 82.7,
      "min": 82.69,
      "max": 82.71,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.13,
      "min": 2.13,
      "max": 2.14,
      "n": 3
     },
     "completed": [
      128,
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1312.5,
       "min": 1305.0,
       "max": 1320.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      143.8,
      385.6,
      655.2,
      788.5,
      801.8,
      806.1,
      808.4,
      1204.8,
      1225.1,
      1240.2,
      1292.1,
      1325.8,
      1476.4,
      1503.3,
      1560.3,
      1596.4,
      1602.3,
      1668.9,
      1673.0,
      1696.1,
      1706.1,
      1709.8,
      1715.1,
      1751.7,
      1758.9,
      1766.1,
      1770.7,
      1773.1,
      1786.3,
      1791.6,
      1795.2,
      1799.7,
      1800.6,
      1804.6,
      1807.7,
      1813.2,
      1882.6,
      2075.3,
      2208.9,
      2284.2,
      2353.1,
      2372.4,
      2392.4,
      2397.3,
      2947.5,
      2973.0,
      3561.2,
      4117.5,
      4155.7,
      4651.0,
      4694.9
     ],
     "tpot_quantiles_ms": [
      82.2,
      82.3,
      84.5,
      88.7,
      88.7,
      92.9,
      96.4,
      97.0,
      101.1,
      101.8,
      101.9,
      102.0,
      102.5,
      102.5,
      102.6,
      102.7,
      102.7,
      102.7,
      102.7,
      102.8,
      102.9,
      102.9,
      103.0,
      103.0,
      103.0,
      103.1,
      103.2,
      103.3,
      103.4,
      103.5,
      103.6,
      103.9,
      104.2,
      104.3,
      104.6,
      104.9,
      105.0,
      105.2,
      105.4,
      105.8,
      106.0,
      107.8,
      107.9,
      108.5,
      109.4,
      109.8,
      111.1,
      111.8,
      111.8,
      113.7,
      115.4
     ]
    },
    {
     "x": 64.0,
     "output_throughput": {
      "median": 242.49,
      "min": 241.32,
      "max": 245.56,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 10940.97,
      "min": 10335.52,
      "max": 11433.22,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 26120.43,
      "min": 24482.1,
      "max": 27103.54,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 131.42,
      "min": 128.48,
      "max": 133.76,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 259.5,
      "min": 258.25,
      "max": 263.6,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1325.5,
      "min": 1270.12,
      "max": 1427.29,
      "n": 3
     },
     "median_itl_ms": {
      "median": 85.97,
      "min": 85.91,
      "max": 85.97,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 1.89,
      "min": 1.89,
      "max": 1.92,
      "n": 3
     },
     "completed": [
      256,
      256,
      256
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1020.0,
       "min": 1005.0,
       "max": 1020.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.459
     },
     "ttft_quantiles_ms": [
      210.4,
      1288.4,
      1856.5,
      2458.2,
      3594.8,
      4213.4,
      4803.0,
      4912.8,
      4936.6,
      5006.8,
      5372.5,
      5407.8,
      5418.4,
      5511.1,
      5536.7,
      5575.0,
      5769.4,
      5935.3,
      6033.4,
      6588.3,
      7525.8,
      8550.6,
      8973.8,
      9646.0,
      10179.1,
      10939.0,
      11455.9,
      11970.6,
      12504.9,
      13792.4,
      15977.8,
      16217.0,
      16527.4,
      16647.5,
      16728.2,
      16822.6,
      16867.6,
      17154.4,
      18390.7,
      18650.9,
      19139.9,
      19500.9,
      19839.8,
      20338.9,
      21035.6,
      22832.0,
      23128.8,
      23650.6,
      24081.1,
      24979.8,
      27204.3
     ],
     "tpot_quantiles_ms": [
      72.9,
      79.9,
      120.6,
      120.9,
      121.2,
      121.6,
      121.9,
      122.0,
      122.1,
      122.4,
      122.5,
      122.7,
      122.8,
      122.8,
      122.9,
      123.1,
      123.2,
      123.3,
      123.6,
      123.8,
      123.8,
      124.4,
      125.6,
      127.3,
      128.5,
      131.0,
      135.1,
      139.7,
      142.5,
      145.2,
      148.3,
      148.9,
      153.3,
      158.6,
      161.5,
      162.8,
      165.5,
      169.0,
      169.7,
      172.3,
      174.6,
      175.6,
      177.7,
      179.7,
      183.3,
      186.9,
      190.2,
      197.3,
      207.0,
      241.6,
      308.9
     ]
    }
   ]
  },
  "vllm-async": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 13.25,
      "min": 13.12,
      "max": 13.56,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 191.25,
      "min": 189.23,
      "max": 194.35,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 695.16,
      "min": 684.3,
      "max": 705.37,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 60.37,
      "min": 60.36,
      "max": 60.37,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 104.16,
      "min": 104.01,
      "max": 104.17,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 104.72,
      "min": 104.46,
      "max": 104.85,
      "n": 3
     },
     "median_itl_ms": {
      "median": 60.47,
      "min": 60.47,
      "max": 60.5,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.08,
      "min": 0.07,
      "max": 0.09,
      "n": 3
     },
     "request_throughput": {
      "median": 0.1,
      "min": 0.1,
      "max": 0.11,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1267.5,
       "min": 1260.0,
       "max": 1282.5,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.486
     },
     "ttft_quantiles_ms": [
      76.6,
      76.6,
      77.1,
      78.1,
      167.2,
      170.2,
      171.9,
      174.7,
      178.7,
      180.7,
      181.6,
      182.4,
      183.8,
      183.8,
      183.9,
      184.9,
      185.2,
      185.2,
      185.8,
      186.2,
      186.2,
      187.8,
      188.1,
      188.2,
      190.2,
      190.6,
      190.9,
      192.3,
      192.9,
      197.8,
      211.1,
      217.2,
      344.3,
      386.1,
      386.1,
      451.5,
      462.6,
      500.3,
      547.9,
      552.5,
      555.8,
      565.5,
      566.7,
      618.1,
      628.1,
      632.9,
      640.2,
      704.8,
      704.9,
      705.1,
      705.4
     ],
     "tpot_quantiles_ms": [
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.4,
      60.4,
      60.4,
      60.4,
      60.4,
      60.4,
      60.6,
      60.6,
      60.8,
      63.5,
      64.2,
      64.2,
      76.9,
      77.6,
      78.6,
      99.7,
      99.8,
      101.3,
      103.2,
      104.1,
      104.1,
      104.1,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2
     ]
    },
    {
     "x": 2.0,
     "output_throughput": {
      "median": 31.26,
      "min": 31.25,
      "max": 31.26,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 131.62,
      "min": 131.61,
      "max": 132.02,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 147.06,
      "min": 142.56,
      "max": 147.82,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 63.43,
      "min": 63.43,
      "max": 63.44,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 63.55,
      "min": 63.54,
      "max": 63.56,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 64.3,
      "min": 64.15,
      "max": 64.38,
      "n": 3
     },
     "median_itl_ms": {
      "median": 63.42,
      "min": 63.41,
      "max": 63.43,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "request_throughput": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1500.0,
       "min": 1500.0,
       "max": 1507.5,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      79.0,
      79.0,
      79.2,
      79.9,
      128.2,
      128.4,
      128.6,
      128.6,
      128.6,
      128.6,
      128.7,
      128.8,
      128.8,
      128.8,
      128.8,
      128.9,
      128.9,
      128.9,
      129.0,
      129.0,
      129.0,
      129.2,
      129.3,
      129.4,
      129.7,
      129.7,
      133.5,
      134.2,
      134.3,
      134.3,
      134.5,
      134.5,
      134.6,
      134.6,
      134.6,
      139.2,
      139.5,
      139.6,
      139.7,
      139.7,
      139.8,
      139.9,
      139.9,
      140.0,
      140.1,
      140.3,
      140.5,
      140.6,
      142.9,
      148.2,
      149.2
     ],
     "tpot_quantiles_ms": [
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.6,
      63.6
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 61.79,
      "min": 61.78,
      "max": 61.81,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 140.66,
      "min": 140.07,
      "max": 141.08,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 154.49,
      "min": 152.79,
      "max": 156.15,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 64.1,
      "min": 64.09,
      "max": 64.11,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 64.2,
      "min": 64.19,
      "max": 64.21,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 64.93,
      "min": 64.82,
      "max": 64.99,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.11,
      "min": 64.09,
      "max": 64.12,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "request_throughput": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1515.0,
       "min": 1500.0,
       "max": 1545.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      82.5,
      82.5,
      83.6,
      85.4,
      131.6,
      131.6,
      131.9,
      132.2,
      132.2,
      132.4,
      132.6,
      132.7,
      134.0,
      139.5,
      139.6,
      139.6,
      139.9,
      139.9,
      139.9,
      140.1,
      140.1,
      140.2,
      140.2,
      140.4,
      140.4,
      140.5,
      140.6,
      140.7,
      140.8,
      140.8,
      140.9,
      141.0,
      141.0,
      141.1,
      141.1,
      141.2,
      141.4,
      141.4,
      141.5,
      141.8,
      142.2,
      143.1,
      151.9,
      152.5,
      152.8,
      153.6,
      154.2,
      154.5,
      155.3,
      155.8,
      156.2
     ],
     "tpot_quantiles_ms": [
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2
     ]
    },
    {
     "x": 8.0,
     "output_throughput": {
      "median": 112.68,
      "min": 112.58,
      "max": 112.84,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 261.27,
      "min": 250.59,
      "max": 261.68,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1328.81,
      "min": 1311.19,
      "max": 1336.29,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 65.75,
      "min": 65.69,
      "max": 65.85,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 73.53,
      "min": 73.29,
      "max": 73.55,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 67.3,
      "min": 67.08,
      "max": 67.94,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.69,
      "min": 65.65,
      "max": 65.74,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.66,
      "min": 0.61,
      "max": 0.69,
      "n": 3
     },
     "request_throughput": {
      "median": 0.88,
      "min": 0.88,
      "max": 0.88,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1500.0,
       "min": 1470.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      86.1,
      87.0,
      134.6,
      134.9,
      138.1,
      138.7,
      139.7,
      162.2,
      162.8,
      163.1,
      163.6,
      163.7,
      164.0,
      164.4,
      164.9,
      165.3,
      165.9,
      166.5,
      211.3,
      212.0,
      212.5,
      212.6,
      212.8,
      213.2,
      213.8,
      214.3,
      308.2,
      312.6,
      339.2,
      571.6,
      573.2,
      579.9,
      589.5,
      711.3,
      712.5,
      766.0,
      974.5,
      975.5,
      1086.8,
      1087.4,
      1089.1,
      1097.4,
      1098.9,
      1114.7,
      1151.1,
      1152.2,
      1181.1,
      1289.2,
      1290.2,
      1311.3,
      1357.0
     ],
     "tpot_quantiles_ms": [
      65.4,
      65.4,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.5,
      65.6,
      65.6,
      65.6,
      65.6,
      65.6,
      65.6,
      65.7,
      65.8,
      65.9,
      66.6,
      66.9,
      66.9,
      66.9,
      66.9,
      67.0,
      67.0,
      67.0,
      67.6,
      67.6,
      67.9,
      69.9,
      69.9,
      70.5,
      70.5,
      70.9,
      70.9,
      71.2,
      71.7,
      73.0,
      73.5,
      73.8
     ]
    },
    {
     "x": 16.0,
     "output_throughput": {
      "median": 201.37,
      "min": 200.77,
      "max": 201.51,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 284.17,
      "min": 282.56,
      "max": 289.25,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2472.23,
      "min": 2465.48,
      "max": 2550.45,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 69.49,
      "min": 69.46,
      "max": 69.61,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 85.44,
      "min": 85.42,
      "max": 85.98,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 84.86,
      "min": 78.82,
      "max": 86.56,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.5,
      "min": 69.41,
      "max": 69.52,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.01,
      "min": 1.0,
      "max": 1.03,
      "n": 3
     },
     "request_throughput": {
      "median": 1.57,
      "min": 1.57,
      "max": 1.57,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1425.0,
       "min": 1410.0,
       "max": 1455.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      98.9,
      173.9,
      175.2,
      176.5,
      177.7,
      190.9,
      191.9,
      193.0,
      194.4,
      202.8,
      204.8,
      216.4,
      217.5,
      220.2,
      221.8,
      223.6,
      225.6,
      227.3,
      229.6,
      238.1,
      243.1,
      251.7,
      252.6,
      253.0,
      253.9,
      266.4,
      346.4,
      825.1,
      838.6,
      863.1,
      879.7,
      880.8,
      886.5,
      1305.4,
      1309.4,
      1372.0,
      1412.8,
      1473.8,
      1690.7,
      1692.5,
      1695.4,
      1697.7,
      1700.2,
      1775.9,
      1959.3,
      1998.7,
      2026.2,
      2299.9,
      2464.6,
      2506.1,
      2550.7
     ],
     "tpot_quantiles_ms": [
      69.2,
      69.2,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.3,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.4,
      69.5,
      69.5,
      69.5,
      69.6,
      69.6,
      69.6,
      69.9,
      73.7,
      73.8,
      74.2,
      74.2,
      74.2,
      74.2,
      74.9,
      75.0,
      75.0,
      77.2,
      77.2,
      77.5,
      77.9,
      78.4,
      81.2,
      81.3,
      81.3,
      81.3,
      81.7,
      81.7,
      85.5,
      86.1
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 275.3,
      "min": 273.78,
      "max": 275.31,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 1806.54,
      "min": 1769.74,
      "max": 1854.26,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 4815.8,
      "min": 4732.89,
      "max": 4874.22,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 102.1,
      "min": 101.77,
      "max": 102.1,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 113.64,
      "min": 113.56,
      "max": 114.09,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 613.22,
      "min": 611.14,
      "max": 619.43,
      "n": 3
     },
     "median_itl_ms": {
      "median": 80.57,
      "min": 80.57,
      "max": 80.58,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.15,
      "min": 2.14,
      "max": 2.15,
      "n": 3
     },
     "completed": [
      128,
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1275.0,
       "min": 1275.0,
       "max": 1290.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      118.8,
      392.6,
      673.0,
      813.2,
      827.9,
      837.1,
      840.3,
      1259.7,
      1273.9,
      1276.9,
      1330.0,
      1379.4,
      1516.4,
      1571.1,
      1621.5,
      1663.1,
      1671.0,
      1724.4,
      1730.2,
      1735.0,
      1739.3,
      1768.9,
      1775.8,
      1792.8,
      1796.0,
      1806.0,
      1823.1,
      1829.1,
      1831.6,
      1838.0,
      1843.9,
      1857.5,
      1861.4,
      1864.2,
      1869.5,
      1876.9,
      1951.5,
      2147.7,
      2286.2,
      2343.6,
      2400.5,
      2438.2,
      2466.8,
      2474.6,
      3026.9,
      3074.9,
      3663.8,
      4219.0,
      4282.5,
      4733.4,
      4874.7
     ],
     "tpot_quantiles_ms": [
      80.2,
      80.3,
      82.7,
      87.0,
      87.1,
      91.4,
      95.1,
      95.7,
      100.0,
      100.5,
      100.6,
      100.7,
      100.8,
      101.1,
      101.2,
      101.2,
      101.3,
      101.4,
      101.5,
      101.6,
      101.6,
      101.6,
      101.8,
      101.9,
      102.0,
      102.1,
      102.1,
      102.1,
      102.3,
      102.4,
      102.4,
      102.6,
      102.9,
      103.2,
      103.4,
      103.6,
      103.9,
      104.1,
      104.4,
      104.5,
      105.0,
      106.4,
      106.9,
      107.7,
      108.0,
      108.6,
      110.3,
      110.6,
      111.2,
      113.0,
      114.7
     ]
    },
    {
     "x": 64.0,
     "output_throughput": {
      "median": 241.97,
      "min": 241.2,
      "max": 253.32,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 11087.23,
      "min": 9801.72,
      "max": 11174.74,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 25950.32,
      "min": 23667.81,
      "max": 27243.21,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 128.17,
      "min": 126.45,
      "max": 129.68,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 259.59,
      "min": 241.76,
      "max": 267.98,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1127.44,
      "min": 1115.96,
      "max": 1423.48,
      "n": 3
     },
     "median_itl_ms": {
      "median": 83.91,
      "min": 83.62,
      "max": 84.16,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 1.89,
      "min": 1.88,
      "max": 1.98,
      "n": 3
     },
     "completed": [
      256,
      256,
      256
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 997.5,
       "min": 990.0,
       "max": 1005.0,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.44
     },
     "ttft_quantiles_ms": [
      170.6,
      1332.9,
      1918.5,
      2531.4,
      3523.4,
      4341.0,
      4946.7,
      4967.7,
      4989.4,
      5068.1,
      5319.3,
      5484.4,
      5510.1,
      5569.5,
      5589.5,
      5610.0,
      5680.0,
      5895.8,
      6191.0,
      6777.0,
      7030.1,
      8277.8,
      8539.0,
      9337.3,
      9807.3,
      10473.0,
      11095.7,
      11489.5,
      12313.7,
      13381.6,
      15731.5,
      16026.0,
      16334.7,
      16506.3,
      16602.7,
      16652.2,
      16678.0,
      16955.0,
      17411.5,
      18077.5,
      18626.6,
      19053.2,
      19554.9,
      20261.5,
      20870.1,
      22382.1,
      22997.6,
      23643.9,
      24690.0,
      25397.6,
      27265.8
     ],
     "tpot_quantiles_ms": [
      69.2,
      78.6,
      118.8,
      119.2,
      120.1,
      120.6,
      120.8,
      120.9,
      121.0,
      121.2,
      121.3,
      121.5,
      121.7,
      122.1,
      122.2,
      122.3,
      122.4,
      122.6,
      122.6,
      122.8,
      123.6,
      123.7,
      124.0,
      125.3,
      126.3,
      127.2,
      129.5,
      131.1,
      135.0,
      138.2,
      144.1,
      145.6,
      154.3,
      158.7,
      160.9,
      163.2,
      166.4,
      168.2,
      169.1,
      171.4,
      172.4,
      174.9,
      177.8,
      181.4,
      184.8,
      187.7,
      191.3,
      195.8,
      208.1,
      237.9,
      312.9
     ]
    }
   ]
  },
  "vllm-seqs-16": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 13.35,
      "min": 13.11,
      "max": 13.63,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 190.07,
      "min": 187.02,
      "max": 195.64,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 688.21,
      "min": 613.66,
      "max": 696.43,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 60.35,
      "min": 60.35,
      "max": 60.36,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 104.18,
      "min": 104.16,
      "max": 104.2,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 104.57,
      "min": 104.57,
      "max": 104.74,
      "n": 3
     },
     "median_itl_ms": {
      "median": 60.45,
      "min": 60.45,
      "max": 60.46,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.08,
      "min": 0.07,
      "max": 0.08,
      "n": 3
     },
     "request_throughput": {
      "median": 0.1,
      "min": 0.1,
      "max": 0.11,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1275.0,
       "min": 1275.0,
       "max": 1290.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.48
     },
     "ttft_quantiles_ms": [
      76.3,
      76.3,
      76.4,
      77.1,
      172.5,
      173.1,
      174.8,
      177.8,
      181.0,
      183.8,
      183.8,
      184.1,
      184.2,
      184.2,
      184.9,
      184.9,
      185.2,
      185.2,
      186.1,
      186.2,
      186.5,
      186.5,
      187.6,
      187.8,
      188.8,
      189.6,
      189.9,
      190.3,
      193.7,
      196.2,
      197.6,
      211.6,
      227.4,
      359.6,
      359.6,
      420.1,
      425.7,
      427.8,
      475.3,
      475.9,
      536.6,
      583.9,
      593.1,
      611.6,
      611.8,
      614.0,
      642.0,
      643.5,
      649.3,
      704.7,
      705.0
     ],
     "tpot_quantiles_ms": [
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.2,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.4,
      60.4,
      60.4,
      60.6,
      60.7,
      62.2,
      63.1,
      63.4,
      63.4,
      64.0,
      73.4,
      73.6,
      104.0,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2
     ]
    },
    {
     "x": 2.0,
     "output_throughput": {
      "median": 31.26,
      "min": 31.25,
      "max": 31.26,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 132.21,
      "min": 131.63,
      "max": 132.57,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 147.54,
      "min": 142.16,
      "max": 149.38,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 63.43,
      "min": 63.42,
      "max": 63.44,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 63.55,
      "min": 63.52,
      "max": 63.56,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 64.55,
      "min": 64.25,
      "max": 64.59,
      "n": 3
     },
     "median_itl_ms": {
      "median": 63.42,
      "min": 63.41,
      "max": 63.43,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "request_throughput": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.24,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1500.0,
       "min": 1500.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      79.5,
      79.5,
      79.5,
      81.9,
      128.0,
      128.5,
      128.6,
      128.8,
      128.8,
      128.8,
      128.9,
      129.0,
      129.2,
      129.2,
      129.2,
      129.2,
      129.2,
      129.2,
      129.3,
      129.3,
      129.3,
      129.5,
      129.7,
      129.8,
      130.2,
      130.3,
      133.9,
      134.6,
      134.6,
      134.8,
      134.9,
      135.3,
      135.9,
      136.4,
      136.4,
      139.4,
      139.4,
      139.6,
      139.6,
      139.7,
      139.7,
      139.8,
      139.9,
      140.0,
      140.1,
      140.3,
      140.6,
      140.7,
      142.5,
      148.7,
      150.9
     ],
     "tpot_quantiles_ms": [
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.3,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.4,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.5,
      63.6,
      63.6
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 61.58,
      "min": 61.34,
      "max": 61.58,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 142.18,
      "min": 141.53,
      "max": 193.84,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 156.44,
      "min": 153.79,
      "max": 209.61,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 64.32,
      "min": 64.24,
      "max": 64.33,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 64.41,
      "min": 64.4,
      "max": 64.41,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 65.4,
      "min": 65.3,
      "max": 65.61,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.31,
      "min": 64.21,
      "max": 64.32,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "request_throughput": {
      "median": 0.48,
      "min": 0.48,
      "max": 0.48,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1515.0,
       "min": 1500.0,
       "max": 1537.5,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      81.3,
      81.3,
      82.8,
      84.0,
      132.3,
      132.6,
      133.3,
      133.4,
      133.9,
      134.1,
      134.8,
      138.3,
      139.8,
      140.6,
      141.1,
      141.1,
      141.1,
      141.1,
      141.5,
      141.6,
      141.8,
      142.0,
      142.0,
      142.4,
      142.5,
      142.8,
      142.9,
      143.1,
      143.2,
      143.6,
      143.7,
      152.9,
      153.5,
      153.8,
      153.8,
      153.9,
      154.4,
      154.8,
      155.6,
      156.2,
      156.5,
      190.7,
      191.1,
      196.6,
      197.2,
      204.6,
      205.1,
      205.4,
      205.9,
      209.1,
      209.7
     ],
     "tpot_quantiles_ms": [
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4,
      64.4
     ]
    },
    {
     "x": 8.0,
     "output_throughput": {
      "median": 112.47,
      "min": 112.41,
      "max": 112.58,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 259.54,
      "min": 258.41,
      "max": 259.69,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1322.42,
      "min": 1315.26,
      "max": 1367.78,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 65.91,
      "min": 65.89,
      "max": 66.01,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 73.48,
      "min": 73.42,
      "max": 73.52,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 67.25,
      "min": 67.12,
      "max": 67.41,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.89,
      "min": 65.86,
      "max": 65.93,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.66,
      "min": 0.6,
      "max": 0.66,
      "n": 3
     },
     "request_throughput": {
      "median": 0.88,
      "min": 0.88,
      "max": 0.88,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1470.0,
       "max": 1515.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      86.6,
      86.7,
      135.2,
      135.5,
      138.3,
      140.1,
      142.1,
      143.2,
      144.0,
      162.4,
      162.8,
      162.8,
      163.1,
      163.4,
      163.7,
      163.8,
      164.1,
      164.3,
      164.5,
      164.8,
      211.3,
      211.4,
      212.0,
      212.7,
      213.8,
      215.3,
      304.5,
      306.1,
      334.8,
      567.7,
      569.9,
      703.0,
      704.5,
      713.8,
      841.0,
      842.7,
      865.6,
      1082.5,
      1083.5,
      1091.2,
      1092.1,
      1177.7,
      1241.3,
      1261.4,
      1262.5,
      1263.8,
      1272.0,
      1306.6,
      1307.8,
      1349.4,
      1367.9
     ],
     "tpot_quantiles_ms": [
      65.7,
      65.7,
      65.7,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.8,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      66.0,
      66.1,
      66.8,
      66.8,
      67.1,
      67.6,
      69.3,
      69.3,
      69.4,
      69.4,
      70.0,
      70.2,
      70.2,
      70.9,
      71.7,
      73.1,
      73.5,
      73.8
     ]
    },
    {
     "x": 16.0,
     "output_throughput": {
      "median": 200.87,
      "min": 200.61,
      "max": 201.13,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 287.94,
      "min": 285.62,
      "max": 295.66,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2481.61,
      "min": 2459.89,
      "max": 2547.79,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 69.66,
      "min": 69.63,
      "max": 69.82,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 85.56,
      "min": 85.54,
      "max": 85.92,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 80.38,
      "min": 78.77,
      "max": 85.56,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.56,
      "min": 69.44,
      "max": 69.61,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.01,
      "min": 1.01,
      "max": 1.03,
      "n": 3
     },
     "request_throughput": {
      "median": 1.57,
      "min": 1.57,
      "max": 1.57,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1440.0,
       "min": 1425.0,
       "max": 1455.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      101.2,
      185.3,
      186.4,
      187.9,
      190.8,
      192.1,
      192.8,
      204.6,
      205.8,
      207.0,
      208.3,
      218.7,
      219.4,
      220.4,
      221.4,
      222.6,
      223.5,
      224.3,
      226.5,
      230.1,
      243.8,
      245.0,
      263.5,
      264.4,
      266.8,
      294.7,
      351.6,
      836.3,
      839.2,
      842.6,
      879.7,
      908.8,
      911.6,
      1306.6,
      1337.5,
      1371.8,
      1428.8,
      1438.1,
      1691.1,
      1694.0,
      1695.8,
      1717.7,
      1746.4,
      1771.9,
      1973.0,
      2015.5,
      2030.6,
      2371.2,
      2460.2,
      2481.9,
      2548.1
     ],
     "tpot_quantiles_ms": [
      69.2,
      69.3,
      69.3,
      69.3,
      69.3,
      69.4,
      69.4,
      69.4,
      69.4,
      69.5,
      69.5,
      69.5,
      69.5,
      69.6,
      69.6,
      69.6,
      69.6,
      69.6,
      69.6,
      69.6,
      69.6,
      69.6,
      69.6,
      69.7,
      69.7,
      69.7,
      69.7,
      69.9,
      69.9,
      73.6,
      73.9,
      74.1,
      74.1,
      74.2,
      74.2,
      74.9,
      75.1,
      75.1,
      77.0,
      77.2,
      77.3,
      77.8,
      78.3,
      81.1,
      81.2,
      81.3,
      81.5,
      81.7,
      81.8,
      85.5,
      86.4
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 171.78,
      "min": 171.27,
      "max": 172.45,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 12330.02,
      "min": 12300.65,
      "max": 12349.65,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 16060.48,
      "min": 15956.19,
      "max": 16402.26,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 81.14,
      "min": 81.03,
      "max": 81.53,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 103.79,
      "min": 102.41,
      "max": 105.16,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 594.51,
      "min": 587.4,
      "max": 602.86,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.79,
      "min": 69.78,
      "max": 69.82,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.04,
      "min": 0.04,
      "max": 0.05,
      "n": 3
     },
     "request_throughput": {
      "median": 1.34,
      "min": 1.34,
      "max": 1.35,
      "n": 3
     },
     "completed": [
      128,
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1365.0,
       "min": 1335.0,
       "max": 1365.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.22
     },
     "ttft_quantiles_ms": [
      127.1,
      676.6,
      1266.8,
      1275.4,
      1845.9,
      2329.0,
      2361.8,
      11463.6,
      11589.6,
      11692.3,
      11819.2,
      11923.6,
      11976.7,
      12010.9,
      12047.1,
      12130.7,
      12184.6,
      12245.0,
      12251.1,
      12282.4,
      12286.0,
      12292.5,
      12301.2,
      12305.2,
      12310.6,
      12313.0,
      12334.3,
      12351.2,
      12369.2,
      12401.9,
      12434.0,
      12492.9,
      12537.3,
      12680.2,
      12794.1,
      12830.4,
      12871.0,
      12905.1,
      13160.8,
      13250.5,
      13422.0,
      13473.1,
      13611.7,
      13999.1,
      14456.3,
      14750.0,
      15387.0,
      15791.3,
      15870.3,
      15992.0,
      16404.2
     ],
     "tpot_quantiles_ms": [
      70.5,
      73.0,
      77.5,
      77.8,
      77.9,
      78.1,
      78.2,
      78.3,
      78.4,
      78.7,
      78.8,
      78.9,
      79.0,
      79.4,
      79.7,
      80.0,
      80.0,
      80.1,
      80.1,
      80.7,
      80.7,
      80.8,
      80.9,
      81.0,
      81.3,
      81.5,
      81.5,
      81.8,
      82.1,
      82.6,
      82.6,
      82.9,
      83.1,
      83.4,
      84.1,
      84.8,
      85.3,
      86.0,
      86.7,
      87.8,
      88.4,
      89.9,
      93.0,
      93.6,
      94.2,
      94.7,
      95.1,
      96.8,
      98.3,
      102.4,
      113.6
     ]
    },
    {
     "x": 64.0,
     "output_throughput": {
      "median": 138.08,
      "min": 136.87,
      "max": 139.35,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 39889.68,
      "min": 39484.5,
      "max": 40521.88,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 58190.26,
      "min": 56789.37,
      "max": 58350.91,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 103.7,
      "min": 103.5,
      "max": 106.48,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 137.83,
      "min": 132.84,
      "max": 138.09,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1142.95,
      "min": 1086.08,
      "max": 1281.85,
      "n": 3
     },
     "median_itl_ms": {
      "median": 70.56,
      "min": 70.37,
      "max": 70.56,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.02,
      "min": 0.02,
      "max": 0.02,
      "n": 3
     },
     "request_throughput": {
      "median": 1.08,
      "min": 1.07,
      "max": 1.09,
      "n": 3
     },
     "completed": [
      256,
      256,
      256
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 630.0,
       "min": 487.5,
       "max": 915.0,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.656
     },
     "ttft_quantiles_ms": [
      166.6,
      1305.9,
      1908.6,
      2430.1,
      12350.6,
      12929.5,
      13578.8,
      23016.1,
      23955.1,
      24700.9,
      34031.7,
      34304.0,
      34506.8,
      34584.1,
      34780.6,
      34872.0,
      35015.8,
      35121.2,
      35388.4,
      35586.4,
      35842.6,
      36223.6,
      36833.2,
      38173.3,
      39243.7,
      40133.0,
      42081.7,
      43587.6,
      44552.5,
      47252.1,
      48305.4,
      49506.4,
      51345.5,
      52171.1,
      52637.7,
      53358.2,
      54229.2,
      54282.7,
      54512.9,
      54911.2,
      55164.2,
      55349.2,
      55594.9,
      55791.4,
      56278.0,
      56621.8,
      56972.3,
      57499.1,
      57620.7,
      58130.3,
      58539.4
     ],
     "tpot_quantiles_ms": [
      77.1,
      77.7,
      78.1,
      78.2,
      78.7,
      78.9,
      79.5,
      79.9,
      80.1,
      80.6,
      80.8,
      81.3,
      81.5,
      82.0,
      82.5,
      82.9,
      84.0,
      85.1,
      86.3,
      87.8,
      91.5,
      94.3,
      96.5,
      99.1,
      101.6,
      103.7,
      107.3,
      108.9,
      109.4,
      110.9,
      111.7,
      113.3,
      114.4,
      115.0,
      116.1,
      116.7,
      117.4,
      118.4,
      119.0,
      119.7,
      120.7,
      121.4,
      122.5,
      122.8,
      123.0,
      124.3,
      125.4,
      127.4,
      129.0,
      131.0,
      142.4
     ]
    }
   ]
  },
  "vllm-fast-b": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 10.59,
      "min": 10.44,
      "max": 10.69,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 704.63,
      "min": 702.94,
      "max": 704.94,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 706.74,
      "min": 706.13,
      "max": 710.82,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 104.14,
      "min": 104.09,
      "max": 104.15,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 104.19,
      "min": 104.16,
      "max": 104.2,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 104.72,
      "min": 104.53,
      "max": 104.83,
      "n": 3
     },
     "median_itl_ms": {
      "median": 104.06,
      "min": 104.04,
      "max": 104.08,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.03,
      "min": 0.03,
      "max": 0.03,
      "n": 3
     },
     "request_throughput": {
      "median": 0.08,
      "min": 0.08,
      "max": 0.08,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 210.0,
       "min": 210.0,
       "max": 210.0,
       "n": 3
      },
      "max_temp_c": 93.0,
      "thermal_share": 0.787
     },
     "ttft_quantiles_ms": [
      75.5,
      75.5,
      76.8,
      77.1,
      176.5,
      177.6,
      184.5,
      187.8,
      188.4,
      190.6,
      194.1,
      194.1,
      217.0,
      238.9,
      410.5,
      427.3,
      483.9,
      483.9,
      491.2,
      586.7,
      616.6,
      639.5,
      698.1,
      702.7,
      703.1,
      703.4,
      703.8,
      704.5,
      704.6,
      704.6,
      704.8,
      704.9,
      704.9,
      705.1,
      705.1,
      705.1,
      705.4,
      705.4,
      705.7,
      705.9,
      706.0,
      706.0,
      706.1,
      706.1,
      706.6,
      706.9,
      707.0,
      708.2,
      708.3,
      708.8,
      711.2
     ],
     "tpot_quantiles_ms": [
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.4,
      60.4,
      60.4,
      60.5,
      62.3,
      62.5,
      72.5,
      73.7,
      86.1,
      86.1,
      86.1,
      97.8,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 186.68,
      "min": 185.41,
      "max": 187.09,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 4180.76,
      "min": 4131.39,
      "max": 4699.15,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 7319.11,
      "min": 7180.95,
      "max": 8018.23,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 142.67,
      "min": 142.09,
      "max": 148.69,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 205.64,
      "min": 204.06,
      "max": 212.09,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1705.44,
      "min": 1424.28,
      "max": 1788.95,
      "n": 3
     },
     "median_itl_ms": {
      "median": 84.75,
      "min": 83.68,
      "max": 105.78,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.22,
      "min": 0.22,
      "max": 0.22,
      "n": 3
     },
     "request_throughput": {
      "median": 1.46,
      "min": 1.45,
      "max": 1.46,
      "n": 3
     },
     "completed": [
      128,
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 450.0,
       "min": 435.0,
       "max": 562.5,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.694
     },
     "ttft_quantiles_ms": [
      118.8,
      195.3,
      203.7,
      220.4,
      390.0,
      787.5,
      793.9,
      810.6,
      837.1,
      1156.6,
      1414.9,
      1448.1,
      2030.6,
      2037.5,
      2214.3,
      2294.6,
      2535.4,
      2655.9,
      2689.3,
      2839.0,
      3220.2,
      3806.4,
      3933.5,
      4074.6,
      4129.7,
      4180.9,
      4199.5,
      4339.3,
      4528.3,
      4837.1,
      4951.6,
      5106.6,
      5172.2,
      5253.0,
      5356.1,
      5368.0,
      5401.6,
      5464.9,
      5483.4,
      5519.6,
      5699.0,
      5800.2,
      5912.2,
      6041.6,
      6125.1,
      6176.8,
      6514.1,
      6799.7,
      7044.2,
      7316.8,
      8154.4
     ],
     "tpot_quantiles_ms": [
      90.2,
      90.3,
      92.0,
      93.4,
      93.5,
      93.5,
      94.6,
      94.7,
      95.8,
      97.5,
      97.7,
      97.7,
      98.7,
      111.3,
      114.4,
      124.2,
      128.0,
      131.1,
      132.5,
      134.3,
      139.0,
      139.6,
      141.2,
      142.1,
      142.4,
      146.0,
      148.2,
      149.8,
      151.5,
      153.4,
      154.3,
      156.1,
      159.9,
      161.6,
      164.4,
      166.3,
      166.8,
      167.6,
      168.3,
      169.4,
      171.0,
      174.5,
      177.4,
      177.8,
      181.4,
      182.7,
      190.1,
      192.3,
      199.7,
      205.6,
      224.7
     ]
    }
   ]
  },
  "vllm-O0": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 10.47,
      "min": 10.35,
      "max": 10.59,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 707.14,
      "min": 707.04,
      "max": 707.25,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 712.44,
      "min": 711.42,
      "max": 713.46,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 106.48,
      "min": 106.48,
      "max": 106.48,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 106.51,
      "min": 106.5,
      "max": 106.51,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 107.05,
      "min": 106.98,
      "max": 107.13,
      "n": 2
     },
     "median_itl_ms": {
      "median": 106.32,
      "min": 106.31,
      "max": 106.34,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.03,
      "min": 0.03,
      "max": 0.04,
      "n": 2
     },
     "request_throughput": {
      "median": 0.08,
      "min": 0.08,
      "max": 0.08,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 210.0,
       "min": 210.0,
       "max": 210.0,
       "n": 2
      },
      "max_temp_c": 93.0,
      "thermal_share": 0.783
     },
     "ttft_quantiles_ms": [
      89.7,
      89.7,
      95.9,
      95.9,
      184.0,
      184.8,
      184.8,
      186.2,
      186.2,
      186.9,
      193.9,
      193.9,
      210.9,
      303.5,
      303.5,
      353.4,
      353.4,
      506.3,
      510.3,
      510.3,
      528.3,
      653.5,
      653.5,
      706.4,
      706.4,
      707.1,
      707.4,
      707.4,
      707.4,
      707.4,
      707.7,
      707.7,
      707.7,
      708.4,
      708.9,
      708.9,
      709.0,
      709.0,
      709.8,
      710.0,
      710.0,
      710.3,
      710.8,
      710.8,
      711.0,
      711.0,
      711.3,
      711.5,
      711.5,
      712.3,
      713.7
     ],
     "tpot_quantiles_ms": [
      60.9,
      60.9,
      60.9,
      60.9,
      61.0,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.4,
      68.3,
      68.3,
      89.8,
      89.8,
      90.5,
      91.0,
      91.0,
      96.0,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5,
      106.5
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 191.53,
      "min": 190.95,
      "max": 192.11,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 4051.14,
      "min": 4030.96,
      "max": 4071.31,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 7460.57,
      "min": 7447.38,
      "max": 7473.76,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 142.04,
      "min": 141.74,
      "max": 142.35,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 201.42,
      "min": 199.59,
      "max": 203.24,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 1459.42,
      "min": 1354.17,
      "max": 1564.68,
      "n": 2
     },
     "median_itl_ms": {
      "median": 85.61,
      "min": 85.22,
      "max": 85.99,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.22,
      "min": 0.22,
      "max": 0.22,
      "n": 2
     },
     "request_throughput": {
      "median": 1.5,
      "min": 1.49,
      "max": 1.5,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 547.5,
       "min": 487.5,
       "max": 607.5,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.676
     },
     "ttft_quantiles_ms": [
      161.6,
      699.1,
      700.6,
      702.2,
      703.6,
      705.6,
      707.2,
      708.7,
      865.5,
      1120.6,
      1289.6,
      1306.7,
      1884.4,
      2027.6,
      2094.7,
      2377.7,
      2494.2,
      2507.8,
      2612.0,
      2959.7,
      3048.0,
      3889.5,
      3898.9,
      4008.9,
      4023.8,
      4031.7,
      4055.0,
      4072.5,
      4087.7,
      4151.0,
      4232.7,
      4622.1,
      4740.1,
      4817.3,
      4977.4,
      5010.2,
      5168.5,
      5376.7,
      5445.1,
      5471.4,
      5606.1,
      5666.7,
      5691.9,
      5720.2,
      5785.2,
      5977.7,
      6139.1,
      6632.8,
      6653.7,
      7081.3,
      7653.4
     ],
     "tpot_quantiles_ms": [
      88.6,
      89.2,
      90.4,
      91.6,
      92.1,
      95.8,
      95.8,
      95.8,
      95.8,
      96.3,
      96.3,
      96.3,
      98.5,
      109.1,
      111.3,
      120.8,
      122.0,
      126.6,
      128.6,
      131.7,
      133.0,
      133.6,
      134.9,
      138.1,
      141.7,
      142.3,
      142.8,
      144.9,
      145.4,
      148.0,
      148.8,
      150.5,
      152.9,
      159.3,
      160.3,
      161.5,
      162.9,
      163.6,
      165.0,
      165.8,
      168.0,
      171.1,
      172.3,
      176.0,
      176.4,
      177.1,
      181.2,
      187.3,
      191.3,
      199.6,
      215.2
     ]
    }
   ]
  },
  "vllm-O3": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 10.63,
      "min": 10.54,
      "max": 10.72,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 704.76,
      "min": 704.37,
      "max": 705.14,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 709.72,
      "min": 709.53,
      "max": 709.9,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 104.15,
      "min": 104.15,
      "max": 104.16,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 104.19,
      "min": 104.18,
      "max": 104.2,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 104.74,
      "min": 104.61,
      "max": 104.87,
      "n": 2
     },
     "median_itl_ms": {
      "median": 104.07,
      "min": 104.07,
      "max": 104.07,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.03,
      "min": 0.03,
      "max": 0.03,
      "n": 2
     },
     "request_throughput": {
      "median": 0.08,
      "min": 0.08,
      "max": 0.08,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 210.0,
       "min": 210.0,
       "max": 210.0,
       "n": 2
      },
      "max_temp_c": 94.0,
      "thermal_share": 0.784
     },
     "ttft_quantiles_ms": [
      77.0,
      77.0,
      77.1,
      77.1,
      181.3,
      189.3,
      189.3,
      190.4,
      190.4,
      191.3,
      195.7,
      195.7,
      217.6,
      313.2,
      313.2,
      466.9,
      466.9,
      472.2,
      491.7,
      491.7,
      508.8,
      702.9,
      702.9,
      703.3,
      703.3,
      704.2,
      704.5,
      704.5,
      705.0,
      705.0,
      706.0,
      706.3,
      706.3,
      706.3,
      706.9,
      706.9,
      707.0,
      707.0,
      707.3,
      707.5,
      707.5,
      707.5,
      707.6,
      707.6,
      707.7,
      707.7,
      708.1,
      709.4,
      709.4,
      709.9,
      710.0
     ],
     "tpot_quantiles_ms": [
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.3,
      60.4,
      60.4,
      60.4,
      62.6,
      71.2,
      71.2,
      82.4,
      82.4,
      87.1,
      94.6,
      94.6,
      100.6,
      104.1,
      104.1,
      104.1,
      104.1,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2,
      104.2
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 189.86,
      "min": 188.5,
      "max": 191.21,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 4086.24,
      "min": 3988.31,
      "max": 4184.17,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 6681.01,
      "min": 6214.24,
      "max": 7147.77,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 143.85,
      "min": 139.59,
      "max": 148.11,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 201.71,
      "min": 201.48,
      "max": 201.95,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 1601.94,
      "min": 1416.58,
      "max": 1787.29,
      "n": 2
     },
     "median_itl_ms": {
      "median": 94.23,
      "min": 83.87,
      "max": 104.58,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.22,
      "min": 0.22,
      "max": 0.22,
      "n": 2
     },
     "request_throughput": {
      "median": 1.48,
      "min": 1.47,
      "max": 1.49,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 536.25,
       "min": 450.0,
       "max": 622.5,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.691
     },
     "ttft_quantiles_ms": [
      115.8,
      207.1,
      208.8,
      232.2,
      233.4,
      235.4,
      806.0,
      807.5,
      829.2,
      1054.2,
      1429.7,
      1461.5,
      1970.6,
      2054.1,
      2083.8,
      2219.7,
      2640.6,
      2673.5,
      2820.0,
      2875.4,
      2970.9,
      3626.3,
      3793.9,
      3813.5,
      3990.2,
      4134.0,
      4184.0,
      4186.8,
      4232.5,
      4254.2,
      4266.1,
      4443.6,
      4590.1,
      4663.6,
      4758.5,
      4806.3,
      4840.4,
      5111.4,
      5248.5,
      5343.7,
      5353.1,
      5386.4,
      5390.9,
      5553.3,
      5561.0,
      5670.4,
      5791.7,
      6142.2,
      6272.1,
      6645.0,
      7187.5
     ],
     "tpot_quantiles_ms": [
      90.2,
      91.2,
      91.3,
      93.0,
      93.6,
      94.4,
      94.4,
      96.8,
      97.6,
      97.7,
      98.5,
      98.5,
      98.6,
      113.8,
      119.4,
      121.8,
      122.6,
      127.1,
      127.3,
      132.9,
      136.1,
      136.6,
      139.6,
      140.3,
      141.5,
      143.1,
      145.3,
      147.8,
      150.1,
      153.1,
      153.2,
      156.1,
      157.1,
      159.8,
      160.5,
      162.8,
      163.1,
      165.6,
      168.8,
      169.7,
      174.0,
      178.3,
      178.3,
      178.5,
      178.7,
      178.9,
      181.9,
      188.3,
      191.3,
      194.6,
      212.6
     ]
    }
   ]
  },
  "vllm-kv-fp8-triton": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 10.75,
      "min": 10.58,
      "max": 10.92,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 716.98,
      "min": 716.57,
      "max": 717.39,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 721.61,
      "min": 719.07,
      "max": 724.16,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 101.79,
      "min": 101.79,
      "max": 101.8,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 101.83,
      "min": 101.82,
      "max": 101.84,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 102.42,
      "min": 102.35,
      "max": 102.49,
      "n": 2
     },
     "median_itl_ms": {
      "median": 101.7,
      "min": 101.68,
      "max": 101.72,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.03,
      "min": 0.03,
      "max": 0.03,
      "n": 2
     },
     "request_throughput": {
      "median": 0.08,
      "min": 0.08,
      "max": 0.09,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 210.0,
       "min": 210.0,
       "max": 210.0,
       "n": 2
      },
      "max_temp_c": 95.0,
      "thermal_share": 0.793
     },
     "ttft_quantiles_ms": [
      77.3,
      77.3,
      78.9,
      78.9,
      183.0,
      186.5,
      186.5,
      187.8,
      187.8,
      188.6,
      189.7,
      189.7,
      222.4,
      251.3,
      251.3,
      467.0,
      467.0,
      478.6,
      711.8,
      711.8,
      714.9,
      715.7,
      715.7,
      716.4,
      716.4,
      716.4,
      716.7,
      716.7,
      717.0,
      717.0,
      717.1,
      717.2,
      717.2,
      717.3,
      717.5,
      717.5,
      718.4,
      718.4,
      718.7,
      718.8,
      718.8,
      718.9,
      719.1,
      719.1,
      719.2,
      719.2,
      719.3,
      719.9,
      719.9,
      720.0,
      724.9
     ],
     "tpot_quantiles_ms": [
      59.8,
      59.8,
      59.8,
      59.8,
      59.8,
      59.8,
      59.8,
      59.8,
      59.8,
      59.8,
      59.9,
      59.9,
      68.4,
      70.4,
      70.4,
      81.8,
      81.8,
      85.3,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8,
      101.8
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 192.0,
      "min": 183.28,
      "max": 200.72,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 4266.89,
      "min": 4256.41,
      "max": 4277.36,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 8427.18,
      "min": 7099.02,
      "max": 9755.34,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 137.96,
      "min": 131.58,
      "max": 144.34,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 197.87,
      "min": 191.31,
      "max": 204.44,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 1671.77,
      "min": 1602.73,
      "max": 1740.82,
      "n": 2
     },
     "median_itl_ms": {
      "median": 86.18,
      "min": 85.62,
      "max": 86.73,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.22,
      "min": 0.21,
      "max": 0.23,
      "n": 2
     },
     "request_throughput": {
      "median": 1.5,
      "min": 1.43,
      "max": 1.57,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 480.0,
       "min": 480.0,
       "max": 480.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.711
     },
     "ttft_quantiles_ms": [
      154.6,
      714.3,
      715.7,
      717.1,
      718.1,
      718.8,
      720.4,
      721.9,
      732.3,
      1119.7,
      1358.7,
      1943.1,
      1956.5,
      1979.1,
      2084.4,
      2594.2,
      2618.4,
      2774.8,
      2826.4,
      3423.9,
      3453.6,
      3733.5,
      4138.6,
      4232.2,
      4233.4,
      4255.7,
      4316.4,
      4504.3,
      4738.1,
      4778.5,
      4812.4,
      4849.7,
      4864.4,
      4939.0,
      4960.8,
      5087.3,
      5197.8,
      5256.5,
      5264.6,
      5273.9,
      5643.3,
      5925.6,
      6207.0,
      6341.0,
      6524.4,
      6904.9,
      7012.8,
      7100.1,
      8199.1,
      8949.3,
      9755.7
     ],
     "tpot_quantiles_ms": [
      81.3,
      82.9,
      84.5,
      85.3,
      88.3,
      89.1,
      89.1,
      89.1,
      89.9,
      89.9,
      89.9,
      89.9,
      93.7,
      105.8,
      108.4,
      118.7,
      120.8,
      123.3,
      125.0,
      129.2,
      130.5,
      134.7,
      137.3,
      137.5,
      139.0,
      142.0,
      144.2,
      144.3,
      144.4,
      144.5,
      146.0,
      149.2,
      149.4,
      151.4,
      152.8,
      153.4,
      156.8,
      157.2,
      161.3,
      163.7,
      173.7,
      176.6,
      180.0,
      180.2,
      181.4,
      182.1,
      185.7,
      186.1,
      191.3,
      196.2,
      210.2
     ]
    }
   ]
  },
  "vllm-lock-b": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 11.2,
      "min": 10.88,
      "max": 11.51,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 554.3,
      "min": 402.46,
      "max": 706.14,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 710.27,
      "min": 710.0,
      "max": 710.53,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 99.03,
      "min": 93.7,
      "max": 104.35,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 104.4,
      "min": 104.4,
      "max": 104.41,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 105.11,
      "min": 104.92,
      "max": 105.3,
      "n": 2
     },
     "median_itl_ms": {
      "median": 103.96,
      "min": 103.71,
      "max": 104.21,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.04,
      "min": 0.03,
      "max": 0.04,
      "n": 2
     },
     "request_throughput": {
      "median": 0.09,
      "min": 0.09,
      "max": 0.09,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 210.0,
       "min": 210.0,
       "max": 210.0,
       "n": 2
      },
      "max_temp_c": 94.0,
      "thermal_share": 0.687
     },
     "ttft_quantiles_ms": [
      78.4,
      78.4,
      79.7,
      79.7,
      184.7,
      185.9,
      185.9,
      185.9,
      185.9,
      185.9,
      187.8,
      187.8,
      188.8,
      190.0,
      190.0,
      191.6,
      191.6,
      193.0,
      194.3,
      194.3,
      198.0,
      225.1,
      225.1,
      554.5,
      554.5,
      579.8,
      684.9,
      684.9,
      705.1,
      705.1,
      705.9,
      706.4,
      706.4,
      706.4,
      706.6,
      706.6,
      707.7,
      707.7,
      708.1,
      708.2,
      708.2,
      708.2,
      709.0,
      709.0,
      709.3,
      709.3,
      709.7,
      710.1,
      710.1,
      710.2,
      710.6
     ],
     "tpot_quantiles_ms": [
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.2,
      61.2,
      72.2,
      83.6,
      83.6,
      101.4,
      101.4,
      103.8,
      104.3,
      104.3,
      104.3,
      104.3,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4,
      104.4
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 208.77,
      "min": 206.75,
      "max": 210.78,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 3089.84,
      "min": 3088.89,
      "max": 3090.79,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 7107.31,
      "min": 6503.43,
      "max": 7711.18,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 119.87,
      "min": 117.68,
      "max": 122.06,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 207.29,
      "min": 206.9,
      "max": 207.68,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 1405.88,
      "min": 1389.58,
      "max": 1422.18,
      "n": 2
     },
     "median_itl_ms": {
      "median": 84.03,
      "min": 82.91,
      "max": 85.16,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.13,
      "min": 0.1,
      "max": 0.15,
      "n": 2
     },
     "request_throughput": {
      "median": 1.63,
      "min": 1.62,
      "max": 1.65,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 881.25,
       "min": 862.5,
       "max": 900.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.5
     },
     "ttft_quantiles_ms": [
      120.1,
      202.5,
      216.9,
      218.5,
      400.3,
      859.0,
      860.6,
      867.9,
      869.4,
      955.5,
      973.7,
      1512.4,
      1527.7,
      1604.0,
      1606.4,
      2079.4,
      2176.9,
      2179.9,
      2249.5,
      2619.3,
      2681.6,
      2827.1,
      2846.4,
      2861.0,
      2907.1,
      3089.8,
      3124.2,
      3202.3,
      3208.8,
      3228.4,
      3293.7,
      3482.6,
      3485.1,
      3872.1,
      3880.4,
      3933.1,
      3935.1,
      3945.6,
      4149.0,
      4184.3,
      4267.8,
      4286.4,
      4342.5,
      4710.8,
      4809.1,
      5228.5,
      5574.9,
      5672.0,
      5987.7,
      6213.8,
      7981.9
     ],
     "tpot_quantiles_ms": [
      93.0,
      93.1,
      93.2,
      93.6,
      93.7,
      96.7,
      96.7,
      96.7,
      97.2,
      97.2,
      101.2,
      101.7,
      101.7,
      106.0,
      106.1,
      107.4,
      108.3,
      109.3,
      109.4,
      111.5,
      113.2,
      113.9,
      114.7,
      116.0,
      117.7,
      118.5,
      119.8,
      127.9,
      129.0,
      135.1,
      137.9,
      139.3,
      144.3,
      146.2,
      146.8,
      148.6,
      149.4,
      152.1,
      154.4,
      156.8,
      157.7,
      159.7,
      162.6,
      163.2,
      168.4,
      169.7,
      172.9,
      179.6,
      193.8,
      206.9,
      220.9
     ]
    }
   ]
  },
  "vllm-lock-eagle3-k1": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 22.38,
      "min": 21.71,
      "max": 23.06,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 240.02,
      "min": 234.08,
      "max": 245.95,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 662.76,
      "min": 619.22,
      "max": 706.29,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 39.4,
      "min": 39.02,
      "max": 39.79,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 75.92,
      "min": 69.23,
      "max": 82.62,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 99.64,
      "min": 84.47,
      "max": 114.8,
      "n": 2
     },
     "median_itl_ms": {
      "median": 68.32,
      "min": 68.32,
      "max": 68.32,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.16,
      "min": 0.15,
      "max": 0.18,
      "n": 2
     },
     "request_throughput": {
      "median": 0.17,
      "min": 0.17,
      "max": 0.18,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 900.0,
       "min": 900.0,
       "max": 900.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.231
     },
     "ttft_quantiles_ms": [
      84.2,
      84.2,
      85.2,
      85.2,
      183.2,
      183.6,
      183.6,
      183.7,
      183.7,
      183.9,
      184.8,
      184.8,
      186.2,
      187.5,
      187.5,
      189.3,
      189.3,
      189.6,
      189.9,
      189.9,
      223.8,
      228.3,
      228.3,
      244.3,
      244.3,
      245.4,
      245.4,
      245.4,
      245.7,
      245.7,
      246.2,
      246.2,
      246.2,
      247.3,
      247.5,
      247.5,
      249.9,
      249.9,
      251.3,
      251.9,
      251.9,
      260.2,
      386.6,
      386.6,
      515.2,
      515.2,
      571.0,
      637.6,
      637.6,
      704.8,
      706.6
     ],
     "tpot_quantiles_ms": [
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.4,
      68.4,
      68.4,
      68.4,
      68.4,
      68.4,
      68.4,
      68.6,
      80.2,
      80.2,
      80.8,
      80.8,
      80.9,
      84.2,
      84.2,
      105.8,
      111.7
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 271.11,
      "min": 261.21,
      "max": 281.01,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 1004.38,
      "min": 929.24,
      "max": 1079.52,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 3093.23,
      "min": 3016.36,
      "max": 3170.09,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 100.43,
      "min": 94.09,
      "max": 106.78,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 158.56,
      "min": 150.07,
      "max": 167.05,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 817.83,
      "min": 784.12,
      "max": 851.54,
      "n": 2
     },
     "median_itl_ms": {
      "median": 106.23,
      "min": 102.4,
      "max": 110.06,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.01,
      "min": 0.0,
      "max": 0.02,
      "n": 2
     },
     "request_throughput": {
      "median": 2.12,
      "min": 2.04,
      "max": 2.2,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 900.0,
       "min": 900.0,
       "max": 900.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.447
     },
     "ttft_quantiles_ms": [
      126.1,
      234.5,
      285.0,
      286.5,
      438.0,
      450.8,
      480.9,
      534.2,
      593.5,
      603.5,
      668.8,
      698.5,
      731.0,
      748.3,
      779.1,
      803.8,
      890.3,
      891.4,
      892.9,
      899.0,
      903.3,
      928.3,
      944.9,
      948.6,
      1006.5,
      1039.8,
      1059.2,
      1071.9,
      1125.9,
      1176.2,
      1205.8,
      1215.7,
      1223.4,
      1281.8,
      1421.5,
      1431.8,
      1530.4,
      1569.1,
      1628.1,
      1696.6,
      1755.9,
      1863.2,
      1874.1,
      1908.1,
      2023.5,
      2064.7,
      2259.4,
      2277.8,
      2321.9,
      3015.5,
      3250.3
     ],
     "tpot_quantiles_ms": [
      78.9,
      112.4,
      122.7,
      127.1,
      131.5,
      133.0,
      137.8,
      138.6,
      139.6,
      142.4,
      145.2,
      147.5,
      150.1,
      152.1,
      154.5,
      155.8,
      156.1,
      157.9,
      159.8,
      162.7,
      163.9,
      164.9,
      166.9,
      168.3,
      170.3,
      171.0,
      172.3,
      174.2,
      176.5,
      179.1,
      180.5,
      185.2,
      188.3,
      190.1,
      195.6,
      198.6,
      201.7,
      206.1,
      207.7,
      211.3,
      215.1,
      223.7,
      228.2,
      233.5,
      243.7,
      249.6,
      254.7,
      259.8,
      271.0,
      277.0,
      307.4
     ]
    }
   ]
  },
  "vllm-lock-eagle3-k3": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 31.31,
      "min": 31.19,
      "max": 31.44,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 255.08,
      "min": 254.13,
      "max": 256.03,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 276.03,
      "min": 273.38,
      "max": 278.68,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 27.81,
      "min": 26.8,
      "max": 28.83,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 60.78,
      "min": 58.65,
      "max": 62.92,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 77.56,
      "min": 77.51,
      "max": 77.62,
      "n": 2
     },
     "median_itl_ms": {
      "median": 73.88,
      "min": 73.88,
      "max": 73.88,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.25,
      "n": 2
     },
     "request_throughput": {
      "median": 0.24,
      "min": 0.24,
      "max": 0.25,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 900.0,
       "min": 900.0,
       "max": 900.0,
       "n": 2
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      83.3,
      83.3,
      85.9,
      85.9,
      184.2,
      185.4,
      185.4,
      186.5,
      186.5,
      186.7,
      186.8,
      186.8,
      187.5,
      187.8,
      187.8,
      189.7,
      189.7,
      222.7,
      251.8,
      251.8,
      253.3,
      253.4,
      253.4,
      253.7,
      253.7,
      254.6,
      254.8,
      254.8,
      255.7,
      255.7,
      256.2,
      256.4,
      256.4,
      256.5,
      256.9,
      256.9,
      258.2,
      258.2,
      258.3,
      258.8,
      258.8,
      259.0,
      259.5,
      259.5,
      260.0,
      260.0,
      262.2,
      263.5,
      263.5,
      275.7,
      281.4
     ],
     "tpot_quantiles_ms": [
      73.9,
      73.9,
      73.9,
      73.9,
      73.9,
      73.9,
      73.9,
      73.9,
      73.9,
      73.9,
      73.9,
      73.9,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.0,
      74.1,
      74.1,
      74.1,
      74.1
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 333.64,
      "min": 327.82,
      "max": 339.45,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 890.48,
      "min": 838.97,
      "max": 941.99,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 3254.86,
      "min": 3233.28,
      "max": 3276.44,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 77.84,
      "min": 76.49,
      "max": 79.18,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 156.61,
      "min": 156.5,
      "max": 156.72,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 771.47,
      "min": 763.4,
      "max": 779.55,
      "n": 2
     },
     "median_itl_ms": {
      "median": 133.48,
      "min": 133.04,
      "max": 133.92,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 2
     },
     "request_throughput": {
      "median": 2.61,
      "min": 2.56,
      "max": 2.65,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 900.0,
       "min": 900.0,
       "max": 900.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.352
     },
     "ttft_quantiles_ms": [
      124.2,
      272.9,
      274.6,
      277.6,
      279.4,
      479.8,
      482.2,
      488.1,
      621.8,
      625.1,
      629.0,
      633.3,
      641.1,
      662.7,
      717.1,
      770.7,
      774.3,
      778.2,
      780.8,
      782.6,
      797.2,
      839.1,
      869.8,
      896.6,
      933.1,
      935.6,
      939.4,
      940.5,
      941.3,
      1040.8,
      1088.5,
      1120.1,
      1128.0,
      1141.9,
      1153.3,
      1196.1,
      1208.0,
      1348.5,
      1366.3,
      1394.9,
      1434.3,
      1483.5,
      1567.5,
      1629.9,
      1635.0,
      1756.9,
      1950.1,
      2327.3,
      2332.3,
      3031.5,
      3367.9
     ],
     "tpot_quantiles_ms": [
      107.0,
      128.8,
      137.9,
      152.2,
      159.7,
      165.8,
      173.1,
      175.9,
      180.5,
      182.9,
      185.3,
      185.9,
      190.9,
      193.6,
      196.8,
      199.4,
      201.7,
      202.7,
      203.5,
      204.4,
      206.0,
      206.7,
      208.1,
      209.0,
      210.5,
      212.0,
      213.9,
      215.7,
      219.5,
      222.4,
      224.6,
      226.3,
      229.4,
      235.6,
      241.8,
      243.9,
      248.5,
      256.6,
      261.7,
      270.2,
      272.1,
      276.8,
      286.3,
      295.3,
      298.0,
      300.7,
      305.8,
      317.8,
      324.4,
      330.0,
      357.1
     ]
    }
   ]
  },
  "vllm-lock-eagle3-k5": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 31.73,
      "min": 30.61,
      "max": 32.85,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 266.79,
      "min": 266.62,
      "max": 266.96,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 340.63,
      "min": 274.89,
      "max": 406.37,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 26.21,
      "min": 25.11,
      "max": 27.3,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 78.95,
      "min": 63.68,
      "max": 94.22,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 109.63,
      "min": 83.27,
      "max": 135.99,
      "n": 2
     },
     "median_itl_ms": {
      "median": 79.6,
      "min": 79.6,
      "max": 79.61,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.24,
      "min": 0.22,
      "max": 0.26,
      "n": 2
     },
     "request_throughput": {
      "median": 0.25,
      "min": 0.24,
      "max": 0.26,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 900.0,
       "min": 900.0,
       "max": 900.0,
       "n": 2
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.136
     },
     "ttft_quantiles_ms": [
      84.2,
      84.2,
      84.9,
      84.9,
      185.0,
      188.0,
      188.0,
      188.0,
      188.0,
      188.8,
      189.6,
      189.6,
      190.4,
      191.1,
      191.1,
      192.2,
      192.2,
      223.4,
      265.6,
      265.6,
      265.6,
      266.5,
      266.5,
      266.6,
      266.6,
      266.8,
      266.8,
      266.8,
      267.2,
      267.2,
      267.7,
      267.7,
      267.7,
      268.1,
      268.8,
      268.8,
      268.8,
      268.8,
      269.1,
      269.7,
      269.7,
      272.3,
      273.3,
      273.3,
      275.2,
      275.2,
      285.2,
      286.5,
      286.5,
      292.9,
      426.4
     ],
     "tpot_quantiles_ms": [
      79.6,
      79.6,
      79.6,
      79.6,
      79.6,
      79.6,
      79.6,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.7,
      79.8,
      79.8,
      79.8,
      79.8,
      79.8,
      79.8,
      79.8,
      79.8,
      79.8,
      79.8,
      80.1,
      113.2
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 289.47,
      "min": 280.42,
      "max": 298.53,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 1071.43,
      "min": 1059.7,
      "max": 1083.17,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 3356.6,
      "min": 3345.87,
      "max": 3367.33,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 83.72,
      "min": 78.73,
      "max": 88.72,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 210.29,
      "min": 196.25,
      "max": 224.33,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 915.23,
      "min": 835.83,
      "max": 994.64,
      "n": 2
     },
     "median_itl_ms": {
      "median": 227.96,
      "min": 218.39,
      "max": 237.54,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 2
     },
     "request_throughput": {
      "median": 2.26,
      "min": 2.19,
      "max": 2.33,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 885.0,
       "min": 885.0,
       "max": 885.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.432
     },
     "ttft_quantiles_ms": [
      125.3,
      282.8,
      284.4,
      333.6,
      335.1,
      336.7,
      573.1,
      580.9,
      640.0,
      728.7,
      737.0,
      745.0,
      792.8,
      868.8,
      889.0,
      894.2,
      895.9,
      901.5,
      905.8,
      918.9,
      956.2,
      971.7,
      1005.8,
      1040.5,
      1054.5,
      1080.5,
      1083.7,
      1126.9,
      1170.8,
      1220.2,
      1305.8,
      1313.3,
      1339.0,
      1375.4,
      1435.6,
      1472.2,
      1575.7,
      1625.4,
      1655.4,
      1705.8,
      1841.3,
      1852.8,
      1906.4,
      1957.7,
      1984.0,
      2341.5,
      2363.1,
      2411.6,
      2749.7,
      3119.8,
      3477.2
     ],
     "tpot_quantiles_ms": [
      128.4,
      167.9,
      181.4,
      192.4,
      200.1,
      210.4,
      220.1,
      223.8,
      227.9,
      230.9,
      236.2,
      238.3,
      240.7,
      245.0,
      247.2,
      248.6,
      250.0,
      251.7,
      253.2,
      255.1,
      257.5,
      260.4,
      262.7,
      267.5,
      271.1,
      272.4,
      273.8,
      276.4,
      280.5,
      285.3,
      289.4,
      295.0,
      308.3,
      315.9,
      325.6,
      341.5,
      347.6,
      349.8,
      364.5,
      371.9,
      382.9,
      386.5,
      391.6,
      405.3,
      407.3,
      420.2,
      434.3,
      476.1,
      499.6,
      507.6,
      581.2
     ]
    }
   ]
  },
  "vllm-lock-fp8": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 25.41,
      "min": 24.51,
      "max": 26.31,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 238.62,
      "min": 237.74,
      "max": 239.5,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 644.05,
      "min": 559.03,
      "max": 729.07,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 35.25,
      "min": 35.25,
      "max": 35.25,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 47.57,
      "min": 43.12,
      "max": 52.02,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 48.47,
      "min": 44.54,
      "max": 52.4,
      "n": 2
     },
     "median_itl_ms": {
      "median": 35.26,
      "min": 35.25,
      "max": 35.26,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.2,
      "min": 0.19,
      "max": 0.21,
      "n": 2
     },
     "request_throughput": {
      "median": 0.2,
      "min": 0.19,
      "max": 0.21,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 900.0,
       "min": 900.0,
       "max": 900.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.226
     },
     "ttft_quantiles_ms": [
      49.7,
      49.7,
      50.5,
      50.5,
      235.1,
      235.4,
      235.4,
      235.7,
      235.7,
      236.0,
      236.0,
      236.0,
      236.3,
      236.4,
      236.4,
      236.6,
      236.6,
      237.0,
      237.0,
      237.0,
      237.1,
      237.6,
      237.6,
      237.7,
      237.7,
      237.8,
      237.9,
      237.9,
      238.0,
      238.0,
      238.8,
      240.2,
      240.2,
      240.4,
      241.4,
      241.4,
      254.5,
      254.5,
      257.8,
      268.6,
      268.6,
      278.7,
      289.3,
      289.3,
      437.4,
      437.4,
      580.5,
      703.2,
      703.2,
      716.0,
      731.4
     ],
     "tpot_quantiles_ms": [
      35.1,
      35.1,
      35.1,
      35.1,
      35.1,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.2,
      35.3,
      35.3,
      35.3,
      35.3,
      35.3,
      35.3,
      35.3,
      35.3,
      35.3,
      35.3,
      35.3,
      35.3,
      38.9,
      38.9,
      42.9,
      42.9,
      43.2,
      52.0,
      52.0,
      52.0,
      52.0
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 284.68,
      "min": 272.31,
      "max": 297.06,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 2861.79,
      "min": 2700.73,
      "max": 3022.84,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 6431.78,
      "min": 6324.15,
      "max": 6539.4,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 83.82,
      "min": 81.53,
      "max": 86.11,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 156.31,
      "min": 140.74,
      "max": 171.88,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 1143.45,
      "min": 1027.02,
      "max": 1259.88,
      "n": 2
     },
     "median_itl_ms": {
      "median": 49.48,
      "min": 49.3,
      "max": 49.65,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.3,
      "min": 0.27,
      "max": 0.33,
      "n": 2
     },
     "request_throughput": {
      "median": 2.22,
      "min": 2.13,
      "max": 2.32,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 900.0,
       "min": 900.0,
       "max": 900.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.349
     },
     "ttft_quantiles_ms": [
      94.5,
      158.5,
      194.6,
      196.2,
      354.7,
      412.2,
      988.2,
      989.7,
      998.8,
      1024.8,
      1135.9,
      1278.1,
      1621.8,
      1803.1,
      1823.2,
      1963.8,
      1966.2,
      2376.4,
      2458.7,
      2549.3,
      2639.1,
      2663.9,
      2685.0,
      2700.9,
      2790.8,
      2837.1,
      2865.0,
      3023.4,
      3126.6,
      3330.2,
      3517.1,
      3544.4,
      3579.8,
      3687.2,
      3731.9,
      4127.2,
      4179.7,
      4192.7,
      4406.5,
      4419.2,
      4720.6,
      4761.3,
      4901.2,
      4907.2,
      4952.2,
      5004.1,
      5011.0,
      5022.6,
      5297.4,
      6182.7,
      6691.8
     ],
     "tpot_quantiles_ms": [
      54.9,
      58.9,
      63.4,
      63.5,
      64.4,
      64.7,
      68.1,
      68.1,
      69.3,
      69.3,
      71.1,
      74.2,
      74.2,
      75.4,
      75.5,
      80.3,
      80.4,
      80.4,
      80.5,
      80.5,
      80.6,
      81.4,
      81.5,
      81.7,
      83.4,
      83.5,
      85.8,
      86.0,
      86.0,
      86.1,
      89.5,
      89.7,
      92.1,
      92.2,
      94.1,
      94.4,
      95.5,
      98.8,
      99.0,
      99.8,
      100.7,
      103.5,
      107.5,
      108.1,
      112.4,
      119.5,
      124.9,
      137.4,
      140.7,
      162.2,
      181.3
     ]
    }
   ]
  },
  "vllm-lock-small-pool": {
   "kv_pool_tokens": 6560,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 11.6,
      "min": 10.9,
      "max": 12.3,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 449.14,
      "min": 208.97,
      "max": 689.3,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 705.83,
      "min": 703.92,
      "max": 707.74,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 82.61,
      "min": 61.12,
      "max": 104.09,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 104.1,
      "min": 104.1,
      "max": 104.11,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 104.71,
      "min": 104.65,
      "max": 104.77,
      "n": 2
     },
     "median_itl_ms": {
      "median": 82.66,
      "min": 61.35,
      "max": 103.98,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.05,
      "min": 0.03,
      "max": 0.06,
      "n": 2
     },
     "request_throughput": {
      "median": 0.09,
      "min": 0.09,
      "max": 0.1,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 281.25,
       "min": 210.0,
       "max": 352.5,
       "n": 2
      },
      "max_temp_c": 94.0,
      "thermal_share": 0.682
     },
     "ttft_quantiles_ms": [
      79.5,
      79.5,
      81.4,
      81.4,
      183.8,
      183.9,
      183.9,
      185.1,
      185.1,
      186.2,
      187.1,
      187.1,
      187.3,
      188.1,
      188.1,
      188.6,
      188.6,
      188.7,
      190.5,
      190.5,
      191.2,
      195.5,
      195.5,
      222.5,
      222.5,
      246.1,
      565.3,
      565.3,
      677.6,
      677.6,
      697.5,
      700.6,
      700.6,
      701.0,
      701.7,
      701.7,
      701.8,
      701.8,
      701.9,
      702.0,
      702.0,
      703.1,
      704.1,
      704.1,
      704.1,
      704.1,
      704.9,
      706.3,
      706.3,
      707.4,
      707.8
     ],
     "tpot_quantiles_ms": [
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      61.1,
      71.3,
      71.3,
      88.5,
      103.0,
      103.0,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1,
      104.1
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 80.44,
      "min": 79.83,
      "max": 81.05,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 31567.67,
      "min": 31432.85,
      "max": 31702.48,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 48028.04,
      "min": 47880.42,
      "max": 48175.66,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 120.56,
      "min": 118.87,
      "max": 122.25,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 248.17,
      "min": 245.5,
      "max": 250.84,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 767.86,
      "min": 764.7,
      "max": 771.01,
      "n": 2
     },
     "median_itl_ms": {
      "median": 105.53,
      "min": 104.73,
      "max": 106.32,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.02,
      "min": 0.02,
      "max": 0.02,
      "n": 2
     },
     "request_throughput": {
      "median": 0.63,
      "min": 0.62,
      "max": 0.63,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 247.5,
       "min": 240.0,
       "max": 255.0,
       "n": 2
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.711
     },
     "ttft_quantiles_ms": [
      125.5,
      762.7,
      1403.9,
      1407.1,
      1930.3,
      11306.7,
      11950.2,
      11952.7,
      12476.8,
      15523.0,
      19017.9,
      21611.0,
      22173.0,
      22248.9,
      22515.4,
      22817.9,
      22892.4,
      23313.9,
      23383.8,
      24334.0,
      25373.8,
      25748.0,
      27891.9,
      29562.6,
      30746.5,
      31466.2,
      32038.7,
      33153.0,
      35526.0,
      37557.1,
      38977.3,
      40356.5,
      41128.1,
      42286.0,
      42388.2,
      42905.3,
      43702.1,
      43805.8,
      44023.7,
      44340.7,
      44451.7,
      44644.4,
      44933.8,
      45370.5,
      45974.5,
      46097.3,
      46205.2,
      46327.5,
      46588.6,
      47854.1,
      48294.7
     ],
     "tpot_quantiles_ms": [
      71.3,
      71.9,
      72.9,
      72.9,
      72.9,
      73.2,
      73.9,
      74.4,
      74.8,
      75.1,
      76.6,
      77.4,
      79.6,
      80.5,
      84.4,
      98.4,
      107.3,
      109.7,
      110.8,
      113.5,
      114.4,
      116.1,
      118.9,
      120.2,
      120.6,
      120.9,
      121.8,
      122.7,
      126.4,
      127.0,
      128.1,
      128.5,
      128.5,
      128.8,
      130.7,
      131.4,
      131.7,
      134.2,
      136.1,
      137.5,
      137.6,
      140.9,
      145.0,
      148.1,
      149.1,
      157.1,
      173.9,
      182.7,
      214.3,
      235.5,
      257.2
     ]
    }
   ]
  },
  "vllm-eagle3": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 28.49,
      "min": 28.1,
      "max": 35.9,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 264.8,
      "min": 261.81,
      "max": 267.13,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 657.93,
      "min": 563.49,
      "max": 708.0,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 27.6,
      "min": 22.49,
      "max": 29.03,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 71.34,
      "min": 39.07,
      "max": 82.97,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 96.16,
      "min": 80.7,
      "max": 107.43,
      "n": 3
     },
     "median_itl_ms": {
      "median": 72.53,
      "min": 72.36,
      "max": 72.6,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.22,
      "min": 0.19,
      "max": 0.28,
      "n": 3
     },
     "request_throughput": {
      "median": 0.22,
      "min": 0.22,
      "max": 0.28,
      "n": 3
     },
     "completed": [
      16,
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1395.0,
       "min": 1327.5,
       "max": 1410.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.468
     },
     "ttft_quantiles_ms": [
      81.0,
      81.0,
      82.5,
      82.8,
      180.8,
      188.2,
      188.9,
      189.5,
      190.0,
      190.1,
      190.9,
      193.0,
      193.7,
      198.7,
      223.2,
      256.8,
      260.3,
      260.3,
      260.4,
      260.8,
      261.5,
      261.8,
      262.1,
      262.3,
      263.1,
      263.3,
      263.5,
      266.3,
      270.7,
      274.8,
      276.8,
      308.2,
      385.2,
      435.3,
      435.3,
      480.9,
      481.3,
      482.8,
      553.7,
      556.8,
      557.3,
      564.6,
      579.3,
      644.7,
      647.9,
      648.3,
      653.6,
      656.5,
      658.7,
      663.1,
      715.9
     ],
     "tpot_quantiles_ms": [
      72.0,
      72.0,
      72.0,
      72.1,
      72.1,
      72.1,
      72.2,
      72.2,
      72.2,
      72.2,
      72.2,
      72.2,
      72.2,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.3,
      72.4,
      72.5,
      72.6,
      73.8,
      73.8,
      73.9,
      75.4,
      75.4,
      78.1,
      79.3,
      79.5,
      79.8,
      79.9,
      80.0,
      80.1,
      80.5,
      89.9,
      89.9,
      90.7,
      91.4,
      91.7,
      99.4,
      100.2,
      100.4
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 302.47,
      "min": 290.57,
      "max": 315.4,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 959.76,
      "min": 921.92,
      "max": 1044.14,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2889.47,
      "min": 2873.4,
      "max": 3093.4,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 83.26,
      "min": 76.56,
      "max": 88.85,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 193.29,
      "min": 172.46,
      "max": 196.25,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 850.53,
      "min": 693.49,
      "max": 958.45,
      "n": 3
     },
     "median_itl_ms": {
      "median": 158.45,
      "min": 145.22,
      "max": 162.09,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.36,
      "min": 2.27,
      "max": 2.46,
      "n": 3
     },
     "completed": [
      128,
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 862.5,
       "min": 742.5,
       "max": 892.5,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.526
     },
     "ttft_quantiles_ms": [
      120.5,
      278.3,
      319.8,
      322.0,
      476.2,
      483.1,
      490.0,
      586.8,
      629.9,
      634.4,
      654.6,
      709.6,
      714.2,
      716.3,
      723.6,
      760.9,
      778.5,
      798.8,
      859.5,
      886.3,
      906.4,
      918.5,
      926.6,
      937.5,
      953.5,
      996.1,
      1012.7,
      1026.1,
      1052.3,
      1069.0,
      1131.9,
      1162.8,
      1207.9,
      1229.6,
      1272.0,
      1305.2,
      1359.1,
      1440.1,
      1515.9,
      1532.4,
      1544.4,
      1563.1,
      1580.2,
      1597.7,
      1645.8,
      1895.9,
      2005.1,
      2228.7,
      2356.7,
      2872.6,
      3180.5
     ],
     "tpot_quantiles_ms": [
      110.2,
      131.2,
      146.0,
      155.0,
      161.8,
      169.7,
      172.8,
      176.6,
      180.5,
      182.8,
      185.7,
      191.3,
      192.9,
      194.6,
      199.0,
      201.6,
      203.2,
      205.2,
      207.0,
      209.1,
      213.5,
      215.8,
      219.9,
      222.9,
      225.2,
      227.0,
      231.0,
      236.7,
      241.8,
      247.0,
      255.3,
      260.0,
      265.6,
      268.8,
      276.2,
      279.7,
      284.0,
      288.3,
      292.6,
      295.0,
      298.5,
      306.7,
      315.1,
      321.7,
      337.4,
      348.1,
      359.6,
      379.2,
      386.6,
      399.7,
      447.5
     ]
    }
   ]
  },
  "vllm-eagle3-k5": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 28.88,
      "min": 28.47,
      "max": 29.29,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 285.77,
      "min": 282.86,
      "max": 288.67,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 682.09,
      "min": 677.19,
      "max": 686.98,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 26.4,
      "min": 26.13,
      "max": 26.67,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 92.28,
      "min": 80.58,
      "max": 103.99,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 124.43,
      "min": 111.48,
      "max": 137.38,
      "n": 2
     },
     "median_itl_ms": {
      "median": 78.13,
      "min": 78.13,
      "max": 78.14,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.18,
      "min": 0.16,
      "max": 0.21,
      "n": 2
     },
     "request_throughput": {
      "median": 0.23,
      "min": 0.22,
      "max": 0.23,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1357.5,
       "min": 1320.0,
       "max": 1395.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.462
     },
     "ttft_quantiles_ms": [
      81.8,
      81.8,
      89.2,
      89.2,
      190.2,
      192.2,
      192.2,
      195.6,
      195.6,
      200.1,
      215.9,
      215.9,
      267.3,
      269.0,
      269.0,
      270.4,
      270.4,
      271.1,
      271.7,
      271.7,
      272.4,
      273.4,
      273.4,
      273.6,
      273.6,
      274.4,
      291.4,
      291.4,
      300.0,
      300.0,
      305.0,
      311.3,
      311.3,
      414.0,
      497.8,
      497.8,
      553.9,
      553.9,
      579.5,
      587.9,
      587.9,
      594.4,
      594.6,
      594.6,
      666.7,
      666.7,
      675.5,
      685.7,
      685.7,
      687.2,
      691.8
     ],
     "tpot_quantiles_ms": [
      77.6,
      77.6,
      77.6,
      77.6,
      77.7,
      77.8,
      77.8,
      77.8,
      77.8,
      77.8,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      77.9,
      78.0,
      78.0,
      78.0,
      78.0,
      78.0,
      78.3,
      78.6,
      78.6,
      87.2,
      87.5,
      87.5,
      88.1,
      88.1,
      88.8,
      90.1,
      90.1,
      91.8,
      103.9,
      103.9,
      104.4,
      104.4,
      104.8,
      105.2,
      105.2,
      105.9,
      127.0
     ]
    },
    {
     "x": 32.0,
     "output_throughput": {
      "median": 287.04,
      "min": 285.25,
      "max": 288.83,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 1185.28,
      "min": 1142.79,
      "max": 1227.76,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 3175.38,
      "min": 3121.15,
      "max": 3229.61,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 84.43,
      "min": 79.57,
      "max": 89.3,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 206.17,
      "min": 205.5,
      "max": 206.83,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 1005.37,
      "min": 914.71,
      "max": 1096.03,
      "n": 2
     },
     "median_itl_ms": {
      "median": 207.04,
      "min": 206.64,
      "max": 207.43,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 2
     },
     "request_throughput": {
      "median": 2.24,
      "min": 2.23,
      "max": 2.26,
      "n": 2
     },
     "completed": [
      128,
      128
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 742.5,
       "min": 735.0,
       "max": 750.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.506
     },
     "ttft_quantiles_ms": [
      123.1,
      272.1,
      273.7,
      319.4,
      321.0,
      322.8,
      583.6,
      589.9,
      700.5,
      723.8,
      744.8,
      798.7,
      870.8,
      878.5,
      883.6,
      885.4,
      897.4,
      907.3,
      927.7,
      940.7,
      970.3,
      1037.0,
      1049.5,
      1140.6,
      1150.2,
      1179.5,
      1210.1,
      1234.1,
      1296.5,
      1303.8,
      1336.4,
      1442.8,
      1452.5,
      1464.5,
      1479.2,
      1513.5,
      1549.1,
      1576.6,
      1641.1,
      1770.9,
      1869.5,
      1892.4,
      2002.4,
      2013.0,
      2188.4,
      2233.8,
      2272.8,
      2759.8,
      2835.4,
      2885.6,
      3229.7
     ],
     "tpot_quantiles_ms": [
      143.1,
      172.0,
      180.9,
      188.4,
      204.0,
      210.9,
      217.6,
      219.6,
      228.0,
      229.9,
      232.5,
      235.8,
      239.8,
      242.5,
      246.2,
      248.7,
      250.4,
      256.0,
      258.3,
      260.8,
      264.0,
      265.7,
      268.2,
      271.4,
      276.0,
      280.0,
      282.1,
      291.1,
      294.7,
      299.4,
      309.4,
      314.6,
      323.6,
      333.4,
      344.6,
      349.5,
      355.7,
      360.7,
      366.7,
      373.2,
      379.8,
      383.8,
      393.8,
      400.2,
      403.2,
      416.4,
      430.9,
      440.2,
      449.9,
      478.6,
      518.6
     ]
    }
   ]
  }
 },
 "rate": {
  "vllm": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 0.5,
     "output_throughput": {
      "median": 70.58,
      "min": 67.22,
      "max": 92.33,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 317.04,
      "min": 311.89,
      "max": 330.5,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 474.68,
      "min": 410.52,
      "max": 494.51,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 72.71,
      "min": 70.41,
      "max": 74.05,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 78.16,
      "min": 76.27,
      "max": 79.74,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 214.09,
      "min": 208.17,
      "max": 214.82,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.58,
      "min": 65.37,
      "max": 66.14,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.55,
      "min": 0.53,
      "max": 0.72,
      "n": 3
     },
     "request_throughput": {
      "median": 0.55,
      "min": 0.53,
      "max": 0.72,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1470.0,
       "max": 1485.0,
       "n": 3
      },
      "max_temp_c": 86.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      76.9,
      77.2,
      262.4,
      268.5,
      273.8,
      275.2,
      278.3,
      279.6,
      285.5,
      288.2,
      289.1,
      289.4,
      292.1,
      294.4,
      298.3,
      301.5,
      303.9,
      304.7,
      306.3,
      307.6,
      310.3,
      311.9,
      312.5,
      313.5,
      315.1,
      317.3,
      319.0,
      320.3,
      324.6,
      324.7,
      327.2,
      327.3,
      329.5,
      331.5,
      331.6,
      334.7,
      336.5,
      339.5,
      342.2,
      345.2,
      346.8,
      347.5,
      360.3,
      379.0,
      389.5,
      398.6,
      411.2,
      427.8,
      468.4,
      475.4,
      503.1
     ],
     "tpot_quantiles_ms": [
      60.6,
      62.9,
      65.1,
      65.5,
      65.8,
      66.0,
      66.6,
      67.1,
      67.3,
      67.5,
      67.7,
      67.8,
      68.4,
      68.5,
      68.7,
      69.7,
      69.8,
      69.9,
      70.1,
      70.5,
      71.0,
      71.0,
      71.4,
      71.7,
      72.0,
      72.2,
      72.6,
      72.8,
      73.0,
      73.1,
      73.4,
      73.7,
      73.9,
      74.1,
      74.2,
      74.2,
      74.5,
      74.6,
      75.1,
      75.1,
      75.2,
      75.2,
      75.5,
      75.9,
      76.1,
      76.2,
      76.3,
      76.7,
      77.7,
      78.5,
      80.1
     ]
    },
    {
     "x": 1.0,
     "output_throughput": {
      "median": 123.6,
      "min": 118.36,
      "max": 154.52,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 179.89,
      "min": 179.31,
      "max": 182.03,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 212.56,
      "min": 211.75,
      "max": 221.72,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 68.16,
      "min": 67.04,
      "max": 68.61,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 68.76,
      "min": 68.42,
      "max": 69.54,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 77.41,
      "min": 77.32,
      "max": 79.09,
      "n": 3
     },
     "median_itl_ms": {
      "median": 67.24,
      "min": 66.42,
      "max": 67.72,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.97,
      "min": 0.92,
      "max": 1.21,
      "n": 3
     },
     "request_throughput": {
      "median": 0.97,
      "min": 0.92,
      "max": 1.21,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1470.0,
       "max": 1485.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      75.5,
      77.0,
      146.2,
      149.0,
      153.9,
      154.3,
      154.6,
      158.1,
      159.7,
      160.5,
      161.9,
      162.6,
      164.1,
      166.2,
      167.3,
      167.9,
      169.9,
      170.5,
      171.0,
      172.2,
      174.8,
      175.0,
      175.8,
      177.2,
      178.1,
      180.5,
      182.2,
      183.2,
      184.3,
      185.7,
      187.7,
      188.5,
      191.0,
      192.1,
      192.9,
      195.4,
      196.6,
      199.1,
      200.1,
      201.8,
      202.4,
      202.5,
      203.6,
      204.4,
      207.3,
      208.6,
      209.2,
      211.0,
      212.3,
      213.2,
      222.7
     ],
     "tpot_quantiles_ms": [
      62.3,
      64.3,
      64.7,
      65.1,
      65.5,
      65.5,
      65.6,
      65.6,
      65.9,
      66.1,
      66.4,
      66.4,
      66.5,
      66.6,
      66.8,
      67.0,
      67.1,
      67.2,
      67.4,
      67.6,
      67.7,
      67.7,
      67.9,
      68.0,
      68.0,
      68.0,
      68.1,
      68.1,
      68.1,
      68.2,
      68.2,
      68.2,
      68.2,
      68.3,
      68.3,
      68.4,
      68.4,
      68.4,
      68.5,
      68.6,
      68.6,
      68.7,
      68.7,
      68.8,
      68.9,
      69.0,
      69.1,
      69.2,
      69.3,
      69.5,
      69.5
     ]
    },
    {
     "x": 2.0,
     "output_throughput": {
      "median": 227.82,
      "min": 216.47,
      "max": 231.82,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 232.88,
      "min": 222.03,
      "max": 249.53,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 722.35,
      "min": 673.98,
      "max": 755.14,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 88.7,
      "min": 84.46,
      "max": 93.39,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 109.06,
      "min": 107.28,
      "max": 111.6,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 354.8,
      "min": 351.04,
      "max": 357.97,
      "n": 3
     },
     "median_itl_ms": {
      "median": 78.08,
      "min": 76.21,
      "max": 79.66,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.36,
      "min": 1.09,
      "max": 1.44,
      "n": 3
     },
     "request_throughput": {
      "median": 1.78,
      "min": 1.69,
      "max": 1.81,
      "n": 3
     },
     "completed": [
      60,
      60,
      60
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1417.5,
       "min": 1395.0,
       "max": 1485.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      74.8,
      146.8,
      160.1,
      166.9,
      169.7,
      170.2,
      174.1,
      176.4,
      176.7,
      179.7,
      182.9,
      185.6,
      188.4,
      192.5,
      197.6,
      199.9,
      205.3,
      206.9,
      208.3,
      211.2,
      214.7,
      219.1,
      220.7,
      221.7,
      223.8,
      242.9,
      250.6,
      296.8,
      324.0,
      327.8,
      331.3,
      340.1,
      351.4,
      357.7,
      375.9,
      384.1,
      388.6,
      398.9,
      414.4,
      447.3,
      463.0,
      481.1,
      486.2,
      507.2,
      541.3,
      571.2,
      581.8,
      612.8,
      621.7,
      706.8,
      775.8
     ],
     "tpot_quantiles_ms": [
      69.0,
      71.4,
      72.3,
      73.6,
      73.7,
      74.0,
      74.4,
      75.6,
      75.9,
      76.0,
      76.5,
      77.4,
      78.3,
      79.3,
      80.2,
      80.7,
      81.9,
      82.1,
      82.6,
      84.4,
      84.6,
      85.5,
      86.3,
      86.7,
      87.3,
      88.6,
      89.5,
      90.8,
      91.2,
      92.4,
      92.7,
      93.3,
      94.4,
      95.5,
      96.7,
      97.8,
      98.8,
      99.9,
      101.3,
      102.5,
      103.5,
      104.2,
      105.2,
      106.1,
      106.5,
      107.2,
      107.3,
      107.7,
      109.0,
      111.4,
      111.6
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 286.62,
      "min": 283.67,
      "max": 291.74,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 8303.55,
      "min": 7744.05,
      "max": 9672.44,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 18006.79,
      "min": 16346.78,
      "max": 19649.84,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 126.9,
      "min": 126.61,
      "max": 128.43,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 186.97,
      "min": 174.48,
      "max": 190.9,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 621.28,
      "min": 620.04,
      "max": 627.17,
      "n": 3
     },
     "median_itl_ms": {
      "median": 83.45,
      "min": 83.38,
      "max": 83.55,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.24,
      "min": 2.22,
      "max": 2.28,
      "n": 3
     },
     "completed": [
      120,
      120,
      120
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1245.0,
       "min": 1200.0,
       "max": 1290.0,
       "n": 3
      },
      "max_temp_c": 86.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      76.5,
      306.7,
      379.0,
      434.7,
      469.4,
      555.4,
      616.4,
      687.0,
      790.3,
      831.8,
      881.4,
      959.8,
      988.3,
      1029.9,
      1089.8,
      1143.7,
      1268.7,
      1517.6,
      1865.1,
      7326.4,
      7575.6,
      7712.0,
      7828.4,
      7967.8,
      8189.5,
      8316.8,
      8477.6,
      9150.5,
      9474.3,
      9643.7,
      9734.6,
      9857.6,
      10049.2,
      10361.2,
      11195.6,
      12517.2,
      13611.9,
      14143.5,
      14575.3,
      15259.0,
      15532.3,
      15792.4,
      16112.8,
      16321.5,
      16719.6,
      17257.1,
      17419.6,
      17547.1,
      17883.0,
      18505.7,
      19854.3
     ],
     "tpot_quantiles_ms": [
      82.3,
      84.4,
      87.2,
      89.4,
      90.4,
      94.8,
      98.1,
      101.7,
      102.8,
      106.1,
      107.8,
      110.4,
      113.9,
      116.2,
      119.0,
      122.3,
      123.1,
      124.1,
      124.8,
      125.3,
      125.5,
      126.1,
      126.6,
      127.0,
      127.0,
      127.6,
      127.9,
      128.0,
      128.1,
      128.4,
      128.4,
      129.1,
      129.4,
      129.6,
      129.6,
      130.0,
      131.1,
      131.4,
      131.6,
      132.0,
      132.5,
      132.9,
      133.1,
      133.9,
      134.8,
      135.8,
      136.9,
      138.0,
      158.6,
      176.6,
      202.2
     ]
    },
    {
     "x": 8.0,
     "output_throughput": {
      "median": 261.83,
      "min": 259.38,
      "max": 264.28,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 27510.33,
      "min": 26282.57,
      "max": 27958.75,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 78008.36,
      "min": 75396.54,
      "max": 80381.07,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 125.97,
      "min": 125.65,
      "max": 126.15,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 248.32,
      "min": 242.27,
      "max": 249.96,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1183.77,
      "min": 1088.72,
      "max": 1360.91,
      "n": 3
     },
     "median_itl_ms": {
      "median": 83.83,
      "min": 83.51,
      "max": 84.01,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.05,
      "min": 2.03,
      "max": 2.06,
      "n": 3
     },
     "completed": [
      240,
      240,
      240
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1035.0,
       "min": 1005.0,
       "max": 1035.0,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.38
     },
     "ttft_quantiles_ms": [
      76.2,
      734.9,
      1214.9,
      1657.9,
      1852.0,
      2007.0,
      2230.7,
      2393.1,
      2748.8,
      3497.8,
      13738.5,
      14529.5,
      14707.7,
      14872.7,
      15045.1,
      15161.3,
      15362.9,
      16243.5,
      25747.4,
      26034.3,
      26305.2,
      26760.6,
      26996.3,
      27178.0,
      27424.5,
      27716.9,
      28069.1,
      37717.1,
      38703.3,
      39244.6,
      39590.8,
      39841.1,
      40382.9,
      40865.4,
      42216.8,
      51577.1,
      52568.3,
      53458.3,
      54070.8,
      54852.4,
      55428.7,
      56483.5,
      57292.5,
      59544.7,
      70841.3,
      72793.6,
      74004.3,
      74968.1,
      76577.0,
      78096.6,
      80709.3
     ],
     "tpot_quantiles_ms": [
      85.8,
      94.1,
      108.0,
      119.7,
      119.7,
      119.9,
      120.4,
      120.5,
      121.8,
      122.0,
      122.2,
      122.3,
      122.4,
      122.5,
      122.8,
      123.3,
      123.5,
      123.8,
      124.0,
      124.3,
      124.8,
      125.0,
      125.2,
      125.3,
      125.5,
      125.9,
      126.9,
      127.3,
      128.2,
      129.9,
      131.0,
      133.7,
      137.1,
      139.3,
      141.6,
      143.3,
      148.9,
      154.1,
      160.4,
      162.9,
      164.7,
      167.1,
      169.2,
      171.2,
      174.3,
      177.4,
      180.5,
      186.9,
      206.2,
      231.7,
      274.8
     ]
    },
    {
     "x": 16.0,
     "output_throughput": {
      "median": 205.77,
      "min": 204.75,
      "max": 208.41,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 94322.33,
      "min": 91404.02,
      "max": 94402.18,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 256061.1,
      "min": 249556.02,
      "max": 256733.32,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 185.15,
      "min": 182.17,
      "max": 190.82,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 321.75,
      "min": 317.28,
      "max": 330.38,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1448.33,
      "min": 1448.2,
      "max": 1483.43,
      "n": 3
     },
     "median_itl_ms": {
      "median": 92.5,
      "min": 92.41,
      "max": 104.11,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 1.61,
      "min": 1.6,
      "max": 1.63,
      "n": 3
     },
     "completed": [
      480,
      480,
      480
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 450.0,
       "min": 450.0,
       "max": 465.0,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.721
     },
     "ttft_quantiles_ms": [
      76.9,
      1610.9,
      2545.0,
      3482.4,
      4376.2,
      16505.6,
      17515.6,
      18447.0,
      19460.6,
      31104.9,
      31975.3,
      32713.2,
      33451.6,
      34772.6,
      46655.1,
      47779.7,
      48801.0,
      50348.2,
      63085.3,
      65170.2,
      66854.4,
      69429.3,
      84658.9,
      88073.2,
      90671.4,
      93347.0,
      108123.3,
      112101.2,
      114975.3,
      117247.8,
      121420.9,
      138094.4,
      141296.9,
      143387.9,
      147634.6,
      164914.2,
      168098.6,
      170978.8,
      174153.2,
      190497.3,
      195587.1,
      198004.8,
      201842.5,
      216652.8,
      222954.1,
      226054.7,
      228490.7,
      232261.7,
      250261.5,
      254069.6,
      257728.6
     ],
     "tpot_quantiles_ms": [
      102.8,
      119.3,
      120.6,
      120.9,
      121.1,
      122.3,
      122.7,
      123.0,
      123.7,
      124.0,
      124.1,
      124.3,
      126.2,
      127.7,
      130.5,
      136.6,
      142.2,
      146.9,
      155.3,
      163.9,
      169.0,
      174.1,
      179.7,
      182.2,
      183.9,
      186.3,
      186.6,
      190.8,
      192.7,
      198.2,
      200.1,
      201.7,
      202.1,
      202.8,
      203.9,
      205.3,
      206.6,
      207.1,
      208.2,
      208.4,
      209.0,
      209.6,
      210.9,
      211.4,
      212.4,
      213.1,
      215.4,
      247.2,
      275.7,
      308.4,
      347.1
     ]
    }
   ]
  },
  "sglang": {
   "kv_pool_tokens": 24094,
   "points": [
    {
     "x": 0.5,
     "output_throughput": {
      "median": 70.64,
      "min": 67.19,
      "max": 92.3,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 284.01,
      "min": 277.57,
      "max": 285.09,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 408.4,
      "min": 404.97,
      "max": 471.92,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 74.72,
      "min": 72.12,
      "max": 76.33,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 80.97,
      "min": 80.19,
      "max": 83.31,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 235.01,
      "min": 233.87,
      "max": 235.1,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.93,
      "min": 65.52,
      "max": 66.87,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.55,
      "min": 0.52,
      "max": 0.72,
      "n": 3
     },
     "request_throughput": {
      "median": 0.55,
      "min": 0.52,
      "max": 0.72,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1485.0,
       "max": 1485.0,
       "n": 3
      },
      "max_temp_c": 86.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      70.8,
      71.7,
      235.6,
      248.1,
      249.5,
      250.5,
      252.3,
      252.5,
      256.8,
      257.2,
      258.5,
      259.7,
      262.3,
      263.3,
      266.4,
      266.8,
      270.0,
      270.0,
      271.4,
      272.5,
      273.3,
      274.1,
      274.8,
      277.5,
      278.5,
      282.2,
      283.3,
      284.7,
      285.0,
      287.0,
      287.6,
      288.8,
      289.8,
      291.0,
      293.2,
      297.5,
      302.6,
      305.0,
      305.2,
      305.5,
      306.6,
      309.2,
      312.8,
      315.5,
      335.8,
      344.7,
      370.6,
      394.9,
      412.8,
      463.2,
      475.4
     ],
     "tpot_quantiles_ms": [
      60.4,
      63.4,
      65.4,
      66.0,
      66.5,
      66.7,
      67.3,
      68.0,
      68.1,
      68.3,
      68.6,
      68.8,
      69.4,
      69.6,
      69.8,
      71.0,
      71.2,
      71.5,
      71.9,
      72.3,
      72.8,
      72.9,
      73.2,
      73.4,
      73.7,
      74.1,
      74.7,
      74.8,
      74.9,
      75.4,
      76.0,
      76.2,
      76.2,
      76.3,
      76.4,
      76.5,
      77.0,
      77.5,
      77.7,
      77.7,
      77.8,
      78.6,
      78.9,
      79.1,
      79.1,
      79.4,
      80.1,
      80.2,
      80.3,
      81.6,
      83.7
     ]
    },
    {
     "x": 1.0,
     "output_throughput": {
      "median": 123.31,
      "min": 118.39,
      "max": 153.57,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 165.75,
      "min": 161.51,
      "max": 174.46,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 219.71,
      "min": 204.21,
      "max": 222.27,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 74.01,
      "min": 72.46,
      "max": 74.59,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 76.31,
      "min": 76.16,
      "max": 77.77,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 113.88,
      "min": 112.9,
      "max": 165.71,
      "n": 3
     },
     "median_itl_ms": {
      "median": 68.78,
      "min": 67.29,
      "max": 69.12,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.96,
      "min": 0.92,
      "max": 1.2,
      "n": 3
     },
     "request_throughput": {
      "median": 0.96,
      "min": 0.92,
      "max": 1.2,
      "n": 3
     },
     "completed": [
      32,
      32,
      32
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1500.0,
       "min": 1500.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      71.3,
      71.4,
      117.3,
      131.2,
      135.8,
      137.5,
      139.3,
      140.0,
      141.7,
      142.0,
      143.1,
      143.6,
      145.2,
      146.6,
      147.1,
      149.9,
      154.5,
      157.3,
      157.8,
      158.7,
      161.1,
      161.3,
      162.9,
      164.9,
      165.7,
      166.2,
      168.2,
      170.8,
      173.2,
      175.8,
      178.2,
      178.5,
      179.7,
      182.8,
      185.2,
      186.4,
      186.5,
      187.9,
      188.3,
      190.6,
      191.6,
      191.8,
      193.6,
      195.8,
      199.7,
      201.3,
      202.2,
      204.7,
      208.7,
      215.7,
      225.2
     ],
     "tpot_quantiles_ms": [
      62.5,
      65.3,
      65.9,
      67.3,
      67.7,
      68.2,
      68.6,
      69.0,
      69.2,
      69.6,
      70.0,
      70.1,
      70.2,
      70.4,
      70.8,
      71.1,
      71.2,
      71.6,
      72.3,
      72.5,
      72.6,
      72.7,
      72.8,
      73.2,
      73.5,
      73.5,
      73.8,
      73.8,
      74.0,
      74.0,
      74.2,
      74.2,
      74.3,
      74.5,
      74.6,
      74.9,
      75.0,
      75.2,
      75.5,
      75.6,
      76.1,
      76.1,
      76.1,
      76.2,
      76.3,
      76.3,
      76.6,
      76.7,
      77.0,
      77.4,
      77.8
     ]
    },
    {
     "x": 2.0,
     "output_throughput": {
      "median": 226.29,
      "min": 215.51,
      "max": 230.07,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 220.6,
      "min": 211.47,
      "max": 236.29,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 603.21,
      "min": 539.35,
      "max": 1428.31,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 99.9,
      "min": 92.89,
      "max": 105.89,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 120.38,
      "min": 116.71,
      "max": 122.07,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 386.62,
      "min": 383.92,
      "max": 429.0,
      "n": 3
     },
     "median_itl_ms": {
      "median": 80.39,
      "min": 77.99,
      "max": 82.91,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.88,
      "min": 0.54,
      "max": 1.18,
      "n": 3
     },
     "request_throughput": {
      "median": 1.77,
      "min": 1.68,
      "max": 1.8,
      "n": 3
     },
     "completed": [
      60,
      60,
      60
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1432.5,
       "min": 1410.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      70.9,
      112.4,
      130.0,
      138.3,
      144.7,
      147.2,
      151.6,
      156.6,
      158.0,
      161.2,
      163.8,
      168.7,
      171.3,
      173.1,
      177.9,
      180.6,
      184.4,
      190.4,
      192.5,
      198.8,
      200.7,
      202.6,
      203.5,
      207.1,
      213.0,
      224.7,
      239.4,
      251.5,
      278.2,
      288.0,
      292.5,
      297.7,
      312.7,
      315.7,
      319.2,
      321.0,
      327.7,
      329.9,
      345.7,
      354.3,
      376.4,
      388.4,
      401.9,
      415.8,
      427.7,
      439.6,
      459.4,
      529.9,
      598.9,
      1170.3,
      1621.4
     ],
     "tpot_quantiles_ms": [
      71.0,
      76.4,
      78.6,
      80.9,
      82.5,
      83.5,
      84.1,
      84.3,
      84.6,
      85.6,
      85.8,
      86.0,
      86.0,
      88.4,
      89.3,
      89.8,
      90.4,
      91.5,
      92.7,
      93.8,
      94.5,
      94.9,
      95.6,
      97.0,
      97.7,
      99.2,
      100.4,
      100.6,
      101.9,
      102.3,
      103.8,
      105.1,
      106.5,
      107.5,
      108.6,
      109.1,
      110.2,
      111.4,
      113.0,
      114.1,
      115.6,
      116.2,
      116.9,
      117.5,
      118.8,
      119.5,
      119.7,
      120.0,
      120.3,
      121.4,
      122.1
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 257.46,
      "min": 254.93,
      "max": 259.72,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 9575.25,
      "min": 8423.14,
      "max": 9843.31,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 24244.88,
      "min": 21501.98,
      "max": 25364.5,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 115.08,
      "min": 113.66,
      "max": 116.93,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 127.23,
      "min": 126.58,
      "max": 128.25,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 680.11,
      "min": 383.16,
      "max": 810.9,
      "n": 3
     },
     "median_itl_ms": {
      "median": 83.71,
      "min": 83.66,
      "max": 83.78,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 2.01,
      "min": 1.99,
      "max": 2.03,
      "n": 3
     },
     "completed": [
      120,
      120,
      120
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1350.0,
       "min": 1320.0,
       "max": 1350.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      71.2,
      274.5,
      336.4,
      389.9,
      416.9,
      524.8,
      593.0,
      645.9,
      713.8,
      761.6,
      812.9,
      865.3,
      928.0,
      1032.2,
      1262.1,
      1648.4,
      7002.1,
      7564.3,
      7716.3,
      7864.4,
      8129.6,
      8424.1,
      8551.8,
      8859.3,
      8999.3,
      9183.2,
      9421.5,
      9809.0,
      10038.5,
      10174.1,
      10515.1,
      12864.8,
      14244.4,
      14554.4,
      14981.9,
      15334.3,
      15775.0,
      16156.4,
      16260.5,
      16435.0,
      16546.7,
      16794.5,
      16980.4,
      17297.0,
      17607.1,
      18518.4,
      20058.0,
      21113.0,
      23818.7,
      24206.8,
      25429.4
     ],
     "tpot_quantiles_ms": [
      66.4,
      68.5,
      72.7,
      76.4,
      93.2,
      95.8,
      99.1,
      100.3,
      100.8,
      101.2,
      101.6,
      106.4,
      108.8,
      109.8,
      110.7,
      110.7,
      111.2,
      113.3,
      113.3,
      113.8,
      114.0,
      114.7,
      115.0,
      115.1,
      115.3,
      115.5,
      116.2,
      116.9,
      118.4,
      119.0,
      119.3,
      120.1,
      120.1,
      120.7,
      121.3,
      121.9,
      122.4,
      123.4,
      123.7,
      124.1,
      124.6,
      125.1,
      125.3,
      125.7,
      125.9,
      126.3,
      126.7,
      126.8,
      127.1,
      127.9,
      128.3
     ]
    },
    {
     "x": 8.0,
     "output_throughput": {
      "median": 253.87,
      "min": 250.66,
      "max": 260.67,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 37210.89,
      "min": 35532.61,
      "max": 37345.35,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 82070.83,
      "min": 81045.76,
      "max": 84231.25,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 105.18,
      "min": 104.57,
      "max": 105.34,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 151.24,
      "min": 149.8,
      "max": 152.97,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 277.35,
      "min": 258.92,
      "max": 316.6,
      "n": 3
     },
     "median_itl_ms": {
      "median": 83.71,
      "min": 83.7,
      "max": 83.8,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 1.98,
      "min": 1.96,
      "max": 2.04,
      "n": 3
     },
     "completed": [
      240,
      240,
      240
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1215.0,
       "min": 1192.5,
       "max": 1230.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.322
     },
     "ttft_quantiles_ms": [
      70.5,
      724.5,
      1045.4,
      1548.2,
      1747.8,
      1931.0,
      2156.3,
      2598.5,
      12334.2,
      13090.6,
      13469.4,
      13740.5,
      13896.5,
      14196.9,
      14430.4,
      14578.8,
      24567.7,
      24921.1,
      25236.6,
      25524.2,
      25746.3,
      25979.2,
      26257.3,
      26706.0,
      35626.3,
      36006.5,
      36471.2,
      37343.8,
      37547.9,
      38010.9,
      38283.0,
      46752.2,
      48282.4,
      48738.5,
      49324.0,
      49679.4,
      50075.6,
      51330.2,
      52676.2,
      60633.6,
      63057.3,
      63645.1,
      64556.4,
      65496.1,
      66100.9,
      67637.4,
      68758.7,
      80080.4,
      81061.8,
      82347.5,
      84592.7
     ],
     "tpot_quantiles_ms": [
      76.9,
      82.0,
      85.3,
      85.4,
      85.7,
      86.5,
      86.8,
      88.1,
      88.7,
      89.0,
      89.9,
      90.5,
      90.8,
      93.0,
      94.5,
      95.4,
      97.2,
      98.0,
      98.2,
      98.3,
      99.2,
      99.7,
      101.6,
      103.6,
      104.0,
      105.2,
      106.0,
      107.1,
      107.6,
      108.6,
      108.7,
      108.8,
      111.3,
      113.8,
      114.3,
      116.3,
      116.6,
      116.9,
      117.9,
      118.1,
      118.7,
      120.2,
      122.4,
      123.9,
      125.7,
      129.8,
      130.1,
      133.8,
      139.3,
      151.2,
      175.5
     ]
    },
    {
     "x": 16.0,
     "output_throughput": {
      "median": 218.24,
      "min": 215.57,
      "max": 228.96,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 97542.26,
      "min": 94794.53,
      "max": 100312.78,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 238041.74,
      "min": 226583.4,
      "max": 242851.07,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 115.2,
      "min": 109.94,
      "max": 115.88,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 188.69,
      "min": 182.16,
      "max": 194.29,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 137.39,
      "min": 119.78,
      "max": 163.36,
      "n": 3
     },
     "median_itl_ms": {
      "median": 84.57,
      "min": 84.44,
      "max": 84.74,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 1.71,
      "min": 1.68,
      "max": 1.79,
      "n": 3
     },
     "completed": [
      480,
      480,
      480
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 600.0,
       "min": 570.0,
       "max": 765.0,
       "n": 3
      },
      "max_temp_c": 88.0,
      "thermal_share": 0.676
     },
     "ttft_quantiles_ms": [
      71.3,
      1579.5,
      2610.5,
      3537.2,
      14917.1,
      15905.5,
      16834.8,
      17653.9,
      29245.8,
      30147.7,
      30900.6,
      31617.1,
      43139.6,
      44097.6,
      44948.1,
      45857.5,
      57597.1,
      58629.0,
      59906.0,
      62164.7,
      73792.1,
      75654.7,
      77921.2,
      81592.4,
      93747.6,
      97211.0,
      99418.4,
      103630.3,
      115285.0,
      118885.1,
      121268.6,
      132669.7,
      138467.8,
      141223.9,
      145194.3,
      155449.4,
      161095.3,
      165371.3,
      168870.4,
      179111.0,
      184337.1,
      188961.7,
      193349.4,
      201248.7,
      208006.6,
      214525.8,
      218388.4,
      224756.0,
      233500.9,
      238495.2,
      243212.4
     ],
     "tpot_quantiles_ms": [
      83.7,
      84.0,
      84.2,
      87.6,
      88.5,
      88.6,
      89.3,
      91.6,
      93.1,
      96.7,
      97.3,
      97.9,
      98.2,
      98.7,
      99.6,
      102.0,
      103.2,
      105.5,
      106.5,
      107.3,
      107.5,
      108.3,
      109.9,
      110.2,
      111.7,
      114.5,
      116.5,
      116.8,
      117.3,
      118.5,
      120.4,
      122.3,
      124.0,
      127.1,
      128.7,
      131.0,
      132.7,
      136.2,
      138.8,
      141.5,
      145.9,
      147.5,
      152.1,
      153.9,
      159.6,
      161.9,
      165.8,
      167.7,
      173.3,
      176.7,
      204.1
     ]
    }
   ]
  }
 },
 "prefix": {
  "vllm": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 0.0,
     "output_throughput": {
      "median": 113.84,
      "min": 113.65,
      "max": 114.33,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 407.88,
      "min": 370.74,
      "max": 444.84,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 484.79,
      "min": 480.58,
      "max": 490.12,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 65.17,
      "min": 64.07,
      "max": 65.74,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 68.38,
      "min": 67.93,
      "max": 68.41,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 133.61,
      "min": 69.14,
      "max": 167.24,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.04,
      "min": 64.04,
      "max": 64.07,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.78,
      "min": 1.78,
      "max": 1.79,
      "n": 3
     },
     "request_throughput": {
      "median": 1.78,
      "min": 1.78,
      "max": 1.79,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1485.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      85.8,
      150.7,
      191.6,
      213.0,
      215.2,
      216.0,
      216.8,
      252.8,
      300.2,
      341.4,
      344.7,
      364.9,
      365.6,
      366.8,
      367.7,
      370.2,
      371.4,
      371.8,
      373.2,
      375.3,
      378.0,
      390.9,
      402.5,
      403.5,
      404.9,
      407.7,
      408.8,
      409.1,
      409.5,
      410.7,
      412.1,
      442.2,
      443.7,
      444.8,
      445.5,
      446.4,
      447.7,
      450.8,
      454.2,
      455.0,
      459.9,
      462.1,
      471.4,
      474.2,
      477.6,
      478.1,
      481.7,
      482.5,
      484.6,
      486.5,
      510.0
     ],
     "tpot_quantiles_ms": [
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      65.1,
      65.1,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.3,
      65.3,
      65.7,
      65.8,
      65.9,
      65.9,
      65.9,
      65.9,
      65.9,
      66.3,
      67.1,
      67.5,
      67.7,
      67.9,
      68.2,
      68.3,
      68.4,
      69.4
     ]
    },
    {
     "x": 256.0,
     "output_throughput": {
      "median": 111.46,
      "min": 111.44,
      "max": 111.67,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 488.04,
      "min": 485.39,
      "max": 490.6,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 615.87,
      "min": 568.09,
      "max": 630.14,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 64.83,
      "min": 64.83,
      "max": 64.83,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 70.1,
      "min": 70.02,
      "max": 70.38,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 127.42,
      "min": 65.94,
      "max": 128.79,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.8,
      "min": 64.78,
      "max": 64.8,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.74,
      "min": 1.74,
      "max": 1.74,
      "n": 3
     },
     "request_throughput": {
      "median": 1.74,
      "min": 1.74,
      "max": 1.74,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1485.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      85.7,
      156.6,
      216.6,
      217.8,
      218.5,
      219.3,
      221.0,
      274.7,
      287.1,
      295.2,
      299.4,
      301.5,
      304.3,
      305.6,
      308.0,
      336.4,
      339.5,
      343.8,
      345.0,
      347.7,
      483.4,
      484.6,
      485.7,
      486.3,
      487.5,
      488.7,
      489.4,
      490.0,
      490.5,
      490.8,
      491.6,
      492.4,
      493.1,
      493.7,
      494.2,
      494.7,
      495.1,
      495.7,
      496.5,
      496.8,
      497.8,
      500.5,
      501.8,
      503.0,
      504.3,
      505.4,
      566.9,
      568.3,
      615.0,
      629.1,
      630.4
     ],
     "tpot_quantiles_ms": [
      64.6,
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.9,
      64.9,
      65.7,
      65.8,
      65.8,
      65.9,
      66.9,
      67.1,
      67.2,
      67.6,
      67.7,
      67.8,
      67.8,
      67.9,
      68.0,
      68.1,
      68.2,
      68.3,
      68.4,
      68.9,
      69.1,
      69.2,
      69.3,
      69.5,
      70.2,
      70.4,
      72.1
     ]
    },
    {
     "x": 1024.0,
     "output_throughput": {
      "median": 105.99,
      "min": 105.73,
      "max": 106.84,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 395.89,
      "min": 394.77,
      "max": 474.55,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1302.59,
      "min": 1011.96,
      "max": 1326.54,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 68.9,
      "min": 66.78,
      "max": 69.04,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 79.91,
      "min": 77.02,
      "max": 80.28,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 189.33,
      "min": 72.72,
      "max": 189.71,
      "n": 3
     },
     "median_itl_ms": {
      "median": 66.92,
      "min": 66.89,
      "max": 67.05,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.52,
      "min": 1.51,
      "max": 1.53,
      "n": 3
     },
     "request_throughput": {
      "median": 1.66,
      "min": 1.65,
      "max": 1.67,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1455.0,
       "min": 1455.0,
       "max": 1470.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      95.4,
      223.3,
      226.7,
      230.5,
      231.7,
      232.8,
      235.9,
      258.0,
      267.1,
      308.2,
      310.0,
      311.1,
      313.4,
      314.5,
      315.6,
      317.7,
      389.1,
      389.8,
      390.7,
      394.3,
      394.5,
      395.3,
      399.3,
      427.8,
      465.5,
      468.5,
      470.8,
      472.2,
      475.1,
      476.6,
      477.9,
      478.9,
      480.1,
      481.1,
      481.9,
      482.7,
      484.7,
      487.6,
      488.6,
      489.3,
      490.8,
      491.8,
      492.7,
      493.5,
      504.3,
      791.5,
      794.9,
      1011.4,
      1022.5,
      1302.5,
      1326.8
     ],
     "tpot_quantiles_ms": [
      66.6,
      66.7,
      66.7,
      66.7,
      66.7,
      66.7,
      66.8,
      66.8,
      66.8,
      67.0,
      67.1,
      67.1,
      67.1,
      67.1,
      67.3,
      67.4,
      67.5,
      67.5,
      67.5,
      67.6,
      67.8,
      67.8,
      68.1,
      68.4,
      68.5,
      68.7,
      68.8,
      68.9,
      69.0,
      69.1,
      69.3,
      69.7,
      69.9,
      70.0,
      70.1,
      70.2,
      70.2,
      70.3,
      70.4,
      70.5,
      70.5,
      71.0,
      71.2,
      71.3,
      71.5,
      71.7,
      72.1,
      72.9,
      75.3,
      79.1,
      83.3
     ]
    },
    {
     "x": 2048.0,
     "output_throughput": {
      "median": 100.18,
      "min": 99.5,
      "max": 100.91,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 409.52,
      "min": 372.74,
      "max": 495.82,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1766.94,
      "min": 1714.65,
      "max": 2147.98,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 72.28,
      "min": 70.59,
      "max": 72.42,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 89.41,
      "min": 81.79,
      "max": 91.78,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 150.77,
      "min": 128.25,
      "max": 165.36,
      "n": 3
     },
     "median_itl_ms": {
      "median": 70.06,
      "min": 69.68,
      "max": 70.29,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.41,
      "min": 1.39,
      "max": 1.45,
      "n": 3
     },
     "request_throughput": {
      "median": 1.57,
      "min": 1.55,
      "max": 1.58,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1455.0,
       "min": 1425.0,
       "max": 1485.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      104.9,
      238.8,
      248.2,
      250.0,
      251.6,
      253.2,
      254.5,
      281.7,
      286.8,
      333.4,
      335.5,
      336.4,
      360.3,
      364.9,
      366.5,
      367.2,
      372.2,
      375.3,
      398.7,
      400.3,
      404.4,
      406.5,
      407.2,
      407.5,
      408.6,
      409.9,
      412.6,
      446.4,
      451.8,
      454.8,
      460.2,
      463.1,
      468.7,
      472.4,
      485.8,
      489.2,
      495.0,
      497.0,
      500.8,
      502.1,
      504.0,
      505.2,
      507.5,
      512.0,
      513.9,
      537.5,
      1246.1,
      1401.1,
      1715.0,
      1766.7,
      2148.3
     ],
     "tpot_quantiles_ms": [
      69.3,
      69.3,
      69.6,
      69.7,
      69.9,
      70.1,
      70.2,
      70.2,
      70.3,
      70.4,
      70.6,
      70.7,
      70.8,
      70.9,
      71.1,
      71.2,
      71.2,
      71.3,
      71.4,
      71.5,
      71.7,
      71.8,
      71.8,
      72.1,
      72.1,
      72.2,
      72.3,
      72.4,
      72.4,
      72.4,
      72.5,
      72.5,
      72.5,
      72.8,
      72.8,
      73.0,
      73.0,
      73.2,
      73.6,
      73.6,
      73.8,
      74.0,
      74.4,
      74.5,
      74.8,
      74.9,
      75.2,
      75.9,
      80.4,
      88.8,
      96.9
     ]
    },
    {
     "x": 4096.0,
     "output_throughput": {
      "median": 86.83,
      "min": 85.57,
      "max": 89.43,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 374.73,
      "min": 362.7,
      "max": 510.66,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 4019.28,
      "min": 1949.66,
      "max": 4037.14,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 80.76,
      "min": 77.36,
      "max": 81.8,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 115.47,
      "min": 99.55,
      "max": 125.63,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 229.66,
      "min": 140.16,
      "max": 231.75,
      "n": 3
     },
     "median_itl_ms": {
      "median": 76.8,
      "min": 76.13,
      "max": 78.0,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.17,
      "min": 1.09,
      "max": 1.19,
      "n": 3
     },
     "request_throughput": {
      "median": 1.36,
      "min": 1.34,
      "max": 1.4,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1410.0,
       "min": 1395.0,
       "max": 1410.0,
       "n": 3
      },
      "max_temp_c": 86.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      116.4,
      255.4,
      269.1,
      282.9,
      286.5,
      288.2,
      290.5,
      294.1,
      301.6,
      303.3,
      312.9,
      315.8,
      328.7,
      332.5,
      333.8,
      339.2,
      341.0,
      342.9,
      344.6,
      354.9,
      363.9,
      374.5,
      380.5,
      454.0,
      458.8,
      461.3,
      467.0,
      468.9,
      469.7,
      470.7,
      471.7,
      472.2,
      473.8,
      493.3,
      494.2,
      496.4,
      503.6,
      510.2,
      512.7,
      514.0,
      515.5,
      518.8,
      522.7,
      756.8,
      1505.2,
      1834.4,
      1900.9,
      2096.0,
      2778.8,
      4019.1,
      4037.5
     ],
     "tpot_quantiles_ms": [
      73.6,
      75.1,
      75.4,
      76.3,
      77.0,
      77.1,
      77.1,
      77.1,
      77.2,
      77.3,
      77.4,
      77.6,
      77.7,
      77.8,
      78.5,
      78.6,
      78.8,
      78.9,
      79.0,
      79.1,
      79.2,
      79.6,
      80.0,
      80.0,
      80.2,
      80.5,
      80.7,
      80.9,
      80.9,
      81.0,
      81.1,
      81.2,
      81.4,
      81.5,
      81.8,
      81.8,
      82.3,
      82.5,
      82.8,
      82.8,
      83.9,
      84.5,
      84.7,
      85.0,
      85.3,
      85.7,
      86.5,
      88.6,
      96.5,
      106.1,
      131.4
     ]
    }
   ]
  },
  "vllm-no-prefix-cache": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 0.0,
     "output_throughput": {
      "median": 113.57,
      "min": 113.48,
      "max": 114.25,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 392.13,
      "min": 373.68,
      "max": 407.57,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 492.09,
      "min": 469.75,
      "max": 510.21,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 65.26,
      "min": 64.99,
      "max": 65.8,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 68.39,
      "min": 68.06,
      "max": 68.43,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 134.68,
      "min": 129.24,
      "max": 172.94,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.06,
      "min": 63.93,
      "max": 64.07,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.77,
      "min": 1.77,
      "max": 1.79,
      "n": 3
     },
     "request_throughput": {
      "median": 1.77,
      "min": 1.77,
      "max": 1.79,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1470.0,
       "max": 1575.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      99.3,
      149.8,
      209.8,
      210.6,
      213.2,
      214.8,
      216.4,
      306.8,
      325.7,
      344.0,
      361.3,
      365.4,
      366.9,
      370.1,
      371.7,
      373.6,
      383.2,
      384.1,
      385.2,
      388.2,
      390.1,
      391.5,
      391.9,
      392.3,
      392.7,
      393.1,
      393.8,
      396.4,
      397.4,
      399.2,
      402.2,
      406.3,
      407.3,
      408.3,
      409.2,
      410.4,
      411.9,
      414.1,
      456.2,
      460.9,
      467.6,
      474.5,
      478.9,
      480.3,
      481.3,
      481.8,
      483.6,
      484.2,
      486.5,
      487.2,
      549.7
     ],
     "tpot_quantiles_ms": [
      63.9,
      63.9,
      63.9,
      63.9,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.0,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      64.1,
      65.0,
      65.0,
      65.0,
      65.0,
      65.0,
      65.0,
      65.1,
      65.1,
      65.1,
      65.2,
      65.2,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.4,
      65.4,
      65.8,
      65.8,
      65.8,
      65.9,
      65.9,
      65.9,
      65.9,
      66.1,
      67.6,
      67.8,
      68.3,
      68.3,
      68.4,
      68.5,
      70.9
     ]
    },
    {
     "x": 256.0,
     "output_throughput": {
      "median": 99.1,
      "min": 98.97,
      "max": 99.56,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 752.77,
      "min": 746.5,
      "max": 827.83,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1096.04,
      "min": 1082.31,
      "max": 1103.02,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 70.17,
      "min": 68.29,
      "max": 70.25,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 77.63,
      "min": 77.29,
      "max": 77.65,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 322.89,
      "min": 255.21,
      "max": 378.79,
      "n": 3
     },
     "median_itl_ms": {
      "median": 65.27,
      "min": 65.27,
      "max": 65.29,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.97,
      "min": 0.97,
      "max": 1.12,
      "n": 3
     },
     "request_throughput": {
      "median": 1.55,
      "min": 1.55,
      "max": 1.56,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1462.5,
       "min": 1455.0,
       "max": 1470.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      176.9,
      292.7,
      297.3,
      298.9,
      299.9,
      301.8,
      305.0,
      731.8,
      734.0,
      738.0,
      739.0,
      740.1,
      740.8,
      741.4,
      745.0,
      746.8,
      747.3,
      747.7,
      748.1,
      749.1,
      750.1,
      751.8,
      755.7,
      758.7,
      768.2,
      770.0,
      813.3,
      824.5,
      825.9,
      827.5,
      828.7,
      833.8,
      835.5,
      1000.8,
      1005.1,
      1007.8,
      1019.0,
      1023.4,
      1033.6,
      1044.3,
      1053.8,
      1056.9,
      1057.7,
      1066.1,
      1067.9,
      1068.7,
      1071.7,
      1073.1,
      1082.1,
      1095.9,
      1103.4
     ],
     "tpot_quantiles_ms": [
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.2,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      65.3,
      66.2,
      66.4,
      67.4,
      68.2,
      68.3,
      68.3,
      68.3,
      68.3,
      68.3,
      68.4,
      68.4,
      69.4,
      69.5,
      69.5,
      70.2,
      70.2,
      70.3,
      70.3,
      70.3,
      70.3,
      70.3,
      70.4,
      70.4,
      70.5,
      70.6,
      71.3,
      71.4,
      71.5,
      71.6,
      76.3,
      76.4,
      77.1,
      77.5,
      77.7,
      78.5,
      79.2
     ]
    },
    {
     "x": 1024.0,
     "output_throughput": {
      "median": 70.88,
      "min": 70.86,
      "max": 70.95,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 1213.27,
      "min": 1212.38,
      "max": 1214.18,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2852.85,
      "min": 2827.56,
      "max": 2864.82,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 94.93,
      "min": 94.77,
      "max": 95.0,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 106.34,
      "min": 106.32,
      "max": 106.48,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 601.7,
      "min": 598.27,
      "max": 603.84,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.02,
      "min": 69.02,
      "max": 69.03,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.05,
      "min": 0.03,
      "max": 0.05,
      "n": 3
     },
     "request_throughput": {
      "median": 1.11,
      "min": 1.11,
      "max": 1.11,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1410.0,
       "min": 1395.0,
       "max": 1410.0,
       "n": 3
      },
      "max_temp_c": 87.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      385.0,
      529.4,
      532.1,
      537.9,
      542.0,
      546.7,
      554.6,
      852.9,
      859.2,
      869.2,
      881.4,
      886.8,
      965.0,
      1173.0,
      1180.3,
      1187.5,
      1190.5,
      1194.3,
      1197.7,
      1203.2,
      1204.9,
      1207.4,
      1210.1,
      1210.6,
      1212.3,
      1213.4,
      1216.0,
      1219.5,
      1222.0,
      1228.0,
      1372.9,
      1386.3,
      1395.5,
      1413.9,
      1504.5,
      1515.7,
      1557.6,
      1582.0,
      1584.6,
      1591.3,
      1604.6,
      1747.2,
      1898.0,
      1918.6,
      1975.4,
      1984.9,
      2174.3,
      2194.8,
      2206.0,
      2795.7,
      2982.5
     ],
     "tpot_quantiles_ms": [
      68.7,
      74.2,
      79.7,
      81.0,
      84.4,
      84.4,
      84.7,
      85.2,
      85.8,
      87.3,
      87.7,
      87.8,
      88.0,
      89.1,
      89.6,
      89.8,
      89.9,
      90.1,
      90.3,
      90.5,
      91.3,
      91.7,
      92.2,
      93.9,
      94.7,
      95.0,
      95.1,
      95.2,
      95.3,
      95.5,
      95.7,
      95.7,
      95.8,
      95.9,
      95.9,
      96.1,
      96.9,
      97.3,
      98.0,
      99.4,
      100.0,
      100.6,
      101.3,
      102.8,
      104.1,
      104.4,
      105.0,
      105.6,
      106.0,
      107.5,
      110.3
     ]
    },
    {
     "x": 2048.0,
     "output_throughput": {
      "median": 46.58,
      "min": 46.17,
      "max": 46.7,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 2549.7,
      "min": 2545.69,
      "max": 2562.85,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 5214.79,
      "min": 5214.5,
      "max": 5393.28,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 119.75,
      "min": 119.36,
      "max": 126.88,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 185.46,
      "min": 180.25,
      "max": 190.89,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1026.05,
      "min": 961.18,
      "max": 1049.22,
      "n": 3
     },
     "median_itl_ms": {
      "median": 74.15,
      "min": 74.1,
      "max": 74.15,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 0.73,
      "min": 0.72,
      "max": 0.73,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1035.0,
       "min": 1027.5,
       "max": 1035.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.252
     },
     "ttft_quantiles_ms": [
      1229.7,
      1387.2,
      1406.5,
      1420.1,
      1432.4,
      1831.6,
      1845.5,
      1961.3,
      1982.0,
      1993.0,
      2009.3,
      2286.2,
      2450.1,
      2480.3,
      2483.7,
      2496.5,
      2511.2,
      2513.6,
      2518.0,
      2521.8,
      2526.0,
      2531.6,
      2539.4,
      2543.0,
      2547.3,
      2550.2,
      2552.2,
      2558.2,
      2562.6,
      2573.5,
      2576.5,
      2582.9,
      2589.5,
      2614.4,
      2665.7,
      2761.8,
      2957.3,
      3193.7,
      3511.2,
      3531.3,
      3633.9,
      3710.9,
      3737.1,
      3780.7,
      4349.0,
      4452.1,
      4628.2,
      4774.5,
      4979.4,
      5169.8,
      5617.8
     ],
     "tpot_quantiles_ms": [
      73.3,
      88.0,
      106.8,
      117.5,
      117.7,
      118.0,
      118.0,
      118.1,
      118.3,
      118.5,
      118.6,
      118.6,
      118.7,
      118.7,
      118.8,
      119.0,
      119.3,
      119.3,
      119.4,
      119.5,
      119.8,
      119.9,
      120.3,
      120.4,
      120.6,
      121.3,
      125.8,
      126.8,
      126.9,
      127.2,
      127.9,
      129.2,
      130.6,
      132.6,
      134.5,
      135.4,
      136.1,
      136.9,
      137.8,
      138.1,
      140.1,
      143.6,
      145.5,
      149.6,
      153.9,
      156.9,
      160.9,
      167.1,
      173.7,
      182.0,
      199.4
     ]
    },
    {
     "x": 4096.0,
     "output_throughput": {
      "median": 19.57,
      "min": 19.5,
      "max": 19.7,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 13353.32,
      "min": 13252.44,
      "max": 13466.36,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 21138.92,
      "min": 21076.34,
      "max": 21499.96,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 192.93,
      "min": 191.82,
      "max": 195.52,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 257.3,
      "min": 245.98,
      "max": 266.42,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 1578.94,
      "min": 1530.64,
      "max": 1589.23,
      "n": 3
     },
     "median_itl_ms": {
      "median": 76.53,
      "min": 76.49,
      "max": 76.54,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 3
     },
     "request_throughput": {
      "median": 0.31,
      "min": 0.3,
      "max": 0.31,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 675.0,
       "min": 660.0,
       "max": 690.0,
       "n": 3
      },
      "max_temp_c": 89.0,
      "thermal_share": 0.671
     },
     "ttft_quantiles_ms": [
      1898.1,
      3195.3,
      4503.0,
      5839.7,
      6067.0,
      6173.7,
      6282.6,
      6676.4,
      6750.6,
      7104.9,
      7152.6,
      9589.5,
      10287.5,
      11469.5,
      11520.6,
      11631.8,
      11660.0,
      11675.1,
      11697.7,
      11751.8,
      11804.1,
      12686.2,
      12885.9,
      12984.7,
      13204.4,
      13349.2,
      13553.5,
      13742.3,
      13800.5,
      13911.6,
      14141.7,
      14660.9,
      14861.3,
      15704.3,
      15916.1,
      16049.1,
      16491.7,
      18270.4,
      18957.0,
      19359.5,
      19501.2,
      19624.8,
      19770.1,
      19992.8,
      20409.9,
      20507.3,
      20565.4,
      20710.8,
      21066.4,
      21468.1,
      21500.3
     ],
     "tpot_quantiles_ms": [
      73.4,
      108.2,
      140.2,
      140.4,
      140.7,
      141.4,
      141.6,
      141.9,
      142.0,
      142.5,
      142.7,
      142.9,
      143.5,
      143.8,
      145.2,
      149.5,
      149.9,
      151.9,
      153.5,
      155.8,
      158.1,
      167.8,
      176.5,
      184.4,
      190.7,
      192.3,
      195.7,
      203.2,
      209.6,
      214.0,
      215.2,
      218.8,
      220.7,
      223.0,
      225.2,
      227.7,
      229.1,
      230.8,
      231.2,
      232.0,
      234.3,
      235.6,
      237.0,
      239.4,
      240.9,
      241.0,
      241.9,
      243.1,
      244.3,
      246.3,
      271.0
     ]
    }
   ]
  },
  "sglang": {
   "kv_pool_tokens": 24094,
   "points": [
    {
     "x": 0.0,
     "output_throughput": {
      "median": 113.5,
      "min": 113.47,
      "max": 113.54,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 353.79,
      "min": 351.25,
      "max": 361.04,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 482.92,
      "min": 475.0,
      "max": 503.28,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 66.1,
      "min": 66.07,
      "max": 66.14,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 70.16,
      "min": 70.1,
      "max": 70.16,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 72.43,
      "min": 71.57,
      "max": 74.91,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.15,
      "min": 64.13,
      "max": 64.16,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.77,
      "min": 1.77,
      "max": 1.77,
      "n": 3
     },
     "request_throughput": {
      "median": 1.77,
      "min": 1.77,
      "max": 1.77,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1470.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 82.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      84.8,
      107.2,
      109.2,
      109.8,
      110.4,
      110.8,
      112.0,
      289.2,
      291.3,
      312.7,
      317.0,
      318.0,
      323.1,
      343.5,
      344.5,
      346.0,
      348.7,
      349.6,
      349.9,
      350.5,
      350.9,
      351.4,
      351.8,
      352.0,
      352.8,
      353.5,
      354.1,
      354.4,
      355.1,
      355.7,
      356.4,
      360.7,
      362.3,
      364.0,
      370.3,
      371.5,
      445.7,
      459.6,
      462.5,
      466.0,
      466.8,
      470.1,
      470.7,
      471.5,
      473.2,
      474.3,
      474.9,
      475.4,
      477.5,
      480.5,
      542.0
     ],
     "tpot_quantiles_ms": [
      64.1,
      64.1,
      64.1,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.2,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      64.3,
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.2,
      66.2,
      66.2,
      66.2,
      66.2,
      66.2,
      66.2,
      66.7,
      66.7,
      66.7,
      66.8,
      67.2,
      67.2,
      68.3,
      69.6,
      69.9,
      70.0,
      70.0,
      70.1,
      70.2
     ]
    },
    {
     "x": 256.0,
     "output_throughput": {
      "median": 110.9,
      "min": 110.79,
      "max": 111.15,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 470.06,
      "min": 323.66,
      "max": 473.83,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 856.32,
      "min": 689.81,
      "max": 900.81,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 67.15,
      "min": 64.93,
      "max": 67.23,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 73.16,
      "min": 72.64,
      "max": 73.3,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 71.56,
      "min": 71.52,
      "max": 71.96,
      "n": 3
     },
     "median_itl_ms": {
      "median": 64.73,
      "min": 64.72,
      "max": 64.75,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.73,
      "min": 1.73,
      "max": 1.74,
      "n": 3
     },
     "request_throughput": {
      "median": 1.73,
      "min": 1.73,
      "max": 1.74,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1485.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      88.4,
      103.7,
      111.9,
      112.8,
      113.7,
      114.4,
      117.2,
      256.6,
      281.5,
      288.6,
      289.9,
      290.6,
      291.4,
      293.5,
      294.6,
      295.4,
      296.8,
      309.5,
      316.0,
      319.0,
      320.2,
      321.4,
      322.4,
      323.5,
      325.6,
      468.6,
      469.3,
      469.7,
      471.9,
      473.3,
      473.9,
      474.5,
      474.9,
      475.6,
      476.9,
      477.7,
      478.5,
      480.0,
      480.5,
      480.6,
      481.0,
      481.2,
      481.5,
      481.9,
      489.4,
      634.9,
      636.0,
      854.7,
      856.2,
      899.5,
      901.0
     ],
     "tpot_quantiles_ms": [
      64.7,
      64.7,
      64.7,
      64.7,
      64.7,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.8,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      64.9,
      67.2,
      67.2,
      67.2,
      67.3,
      67.4,
      67.4,
      67.4,
      67.4,
      67.5,
      67.7,
      67.7,
      67.7,
      67.8,
      67.8,
      67.8,
      67.8,
      67.9,
      67.9,
      68.3,
      70.5,
      70.5,
      70.6,
      70.7,
      70.7,
      70.8,
      77.6
     ]
    },
    {
     "x": 1024.0,
     "output_throughput": {
      "median": 105.65,
      "min": 103.15,
      "max": 105.78,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 466.04,
      "min": 463.78,
      "max": 493.18,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1887.13,
      "min": 1725.09,
      "max": 1968.5,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 66.75,
      "min": 66.74,
      "max": 66.79,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 89.71,
      "min": 89.62,
      "max": 90.87,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 72.95,
      "min": 72.68,
      "max": 73.31,
      "n": 3
     },
     "median_itl_ms": {
      "median": 66.41,
      "min": 66.17,
      "max": 66.64,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.52,
      "min": 1.28,
      "max": 1.52,
      "n": 3
     },
     "request_throughput": {
      "median": 1.65,
      "min": 1.61,
      "max": 1.65,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1440.0,
       "max": 1485.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      89.7,
      116.9,
      118.0,
      120.2,
      121.2,
      124.1,
      127.4,
      215.7,
      218.2,
      219.8,
      222.2,
      257.4,
      259.9,
      263.6,
      437.1,
      460.4,
      461.9,
      462.6,
      463.1,
      463.5,
      464.0,
      464.4,
      464.7,
      464.9,
      465.1,
      465.4,
      465.8,
      466.2,
      466.2,
      466.5,
      466.8,
      467.0,
      467.3,
      467.8,
      468.4,
      469.3,
      470.4,
      491.3,
      493.0,
      494.2,
      494.9,
      495.7,
      501.9,
      503.8,
      657.2,
      1237.0,
      1376.3,
      1677.9,
      1829.9,
      1886.6,
      1968.7
     ],
     "tpot_quantiles_ms": [
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.1,
      66.2,
      66.2,
      66.2,
      66.2,
      66.2,
      66.2,
      66.2,
      66.2,
      66.2,
      66.7,
      66.7,
      66.7,
      66.7,
      66.7,
      66.7,
      66.7,
      66.7,
      66.7,
      66.8,
      66.8,
      66.8,
      66.8,
      66.8,
      66.8,
      66.8,
      66.8,
      66.8,
      66.8,
      68.8,
      69.0,
      69.9,
      70.0,
      70.1,
      70.4,
      70.6,
      70.7,
      71.2,
      71.7,
      72.1,
      72.2,
      72.3,
      72.8,
      85.1,
      87.6,
      96.5
     ]
    },
    {
     "x": 2048.0,
     "output_throughput": {
      "median": 100.38,
      "min": 99.62,
      "max": 102.45,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 440.78,
      "min": 440.16,
      "max": 484.93,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2316.63,
      "min": 2030.26,
      "max": 2685.73,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 70.48,
      "min": 68.62,
      "max": 70.58,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 97.89,
      "min": 95.48,
      "max": 106.5,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 150.62,
      "min": 73.79,
      "max": 152.94,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.1,
      "min": 68.1,
      "max": 69.14,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.4,
      "min": 1.36,
      "max": 1.48,
      "n": 3
     },
     "request_throughput": {
      "median": 1.57,
      "min": 1.56,
      "max": 1.6,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1455.0,
       "min": 1455.0,
       "max": 1470.0,
       "n": 3
      },
      "max_temp_c": 84.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      99.8,
      113.9,
      123.5,
      164.4,
      166.0,
      166.4,
      168.3,
      222.9,
      432.5,
      433.9,
      434.7,
      435.7,
      436.4,
      437.2,
      437.7,
      438.4,
      439.3,
      439.4,
      439.7,
      440.2,
      440.8,
      441.3,
      441.9,
      442.1,
      443.5,
      444.3,
      446.5,
      470.6,
      472.4,
      479.9,
      482.0,
      483.9,
      485.0,
      486.0,
      486.8,
      487.4,
      488.0,
      488.8,
      490.4,
      491.8,
      493.1,
      494.8,
      510.6,
      516.1,
      520.1,
      524.8,
      1638.5,
      1876.6,
      2030.1,
      2684.1,
      2686.1
     ],
     "tpot_quantiles_ms": [
      68.0,
      68.0,
      68.0,
      68.0,
      68.0,
      68.0,
      68.0,
      68.1,
      68.1,
      68.1,
      68.1,
      69.1,
      69.2,
      69.2,
      69.2,
      69.2,
      69.2,
      69.2,
      69.2,
      69.3,
      69.3,
      69.4,
      69.4,
      69.4,
      69.4,
      70.4,
      70.5,
      70.5,
      70.5,
      70.5,
      70.6,
      70.6,
      70.6,
      70.6,
      70.6,
      70.6,
      70.6,
      70.6,
      70.6,
      70.6,
      70.7,
      72.8,
      73.6,
      73.9,
      74.4,
      74.8,
      74.9,
      75.1,
      76.2,
      94.2,
      110.0
     ]
    },
    {
     "x": 4096.0,
     "output_throughput": {
      "median": 88.36,
      "min": 88.29,
      "max": 88.93,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 473.52,
      "min": 446.49,
      "max": 478.8,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 4274.95,
      "min": 3813.85,
      "max": 4792.76,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 75.59,
      "min": 74.94,
      "max": 77.02,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 123.67,
      "min": 123.52,
      "max": 148.0,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 161.53,
      "min": 81.87,
      "max": 214.22,
      "n": 3
     },
     "median_itl_ms": {
      "median": 74.05,
      "min": 73.91,
      "max": 74.16,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.21,
      "min": 1.04,
      "max": 1.22,
      "n": 3
     },
     "request_throughput": {
      "median": 1.38,
      "min": 1.38,
      "max": 1.39,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1440.0,
       "min": 1440.0,
       "max": 1440.0,
       "n": 3
      },
      "max_temp_c": 85.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      116.8,
      163.5,
      178.9,
      188.2,
      204.7,
      216.2,
      220.2,
      363.1,
      369.1,
      375.3,
      378.3,
      404.7,
      405.9,
      408.2,
      409.0,
      411.2,
      413.7,
      415.1,
      452.0,
      454.3,
      457.3,
      471.0,
      472.8,
      474.3,
      475.6,
      477.3,
      478.4,
      479.5,
      480.7,
      483.7,
      564.5,
      578.6,
      581.3,
      582.5,
      583.4,
      585.0,
      586.0,
      586.7,
      587.8,
      588.6,
      591.2,
      592.4,
      597.5,
      612.8,
      2090.9,
      2093.4,
      2564.4,
      2672.5,
      3814.2,
      3943.9,
      4837.6
     ],
     "tpot_quantiles_ms": [
      71.8,
      71.9,
      72.0,
      72.1,
      73.2,
      73.2,
      73.2,
      74.1,
      74.1,
      74.2,
      74.2,
      74.3,
      74.3,
      74.3,
      74.3,
      74.3,
      74.3,
      74.3,
      74.3,
      74.3,
      74.8,
      74.8,
      74.9,
      75.0,
      75.6,
      75.6,
      75.7,
      76.3,
      76.3,
      76.4,
      76.4,
      76.5,
      77.0,
      77.1,
      77.1,
      77.2,
      77.5,
      77.7,
      77.7,
      77.8,
      78.0,
      78.2,
      80.1,
      80.3,
      80.4,
      80.6,
      88.5,
      98.8,
      108.7,
      119.5,
      148.5
     ]
    }
   ]
  }
 },
 "waves": {
  "vllm": {
   "kv_pool_tokens": 24096,
   "points": [
    {
     "x": 0.0,
     "output_throughput": {
      "median": 174.44,
      "min": 164.06,
      "max": 195.01,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 283.85,
      "min": 223.09,
      "max": 284.5,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1162.78,
      "min": 980.33,
      "max": 1304.61,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 74.96,
      "min": 68.44,
      "max": 75.64,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 116.09,
      "min": 103.13,
      "max": 150.94,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 201.88,
      "min": 180.03,
      "max": 215.05,
      "n": 3
     },
     "median_itl_ms": {
      "median": 66.27,
      "min": 66.15,
      "max": 66.39,
      "n": 3
     },
     "goodput_rps": {
      "median": 2.42,
      "min": 2.1,
      "max": 2.61,
      "n": 3
     },
     "request_throughput": {
      "median": 2.65,
      "min": 2.45,
      "max": 2.81,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1492.5,
       "min": 1470.0,
       "max": 1507.5,
       "n": 3
      },
      "max_temp_c": 81.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      92.4,
      196.7,
      203.8,
      204.4,
      204.6,
      204.9,
      205.3,
      205.5,
      205.9,
      206.2,
      206.6,
      207.2,
      207.7,
      208.6,
      209.1,
      210.9,
      212.6,
      213.6,
      214.4,
      214.8,
      216.6,
      220.2,
      225.2,
      230.1,
      269.8,
      272.2,
      278.9,
      283.5,
      286.7,
      290.9,
      306.4,
      311.5,
      313.8,
      323.0,
      333.8,
      339.9,
      348.1,
      372.7,
      392.1,
      647.0,
      648.0,
      648.5,
      649.0,
      649.6,
      650.2,
      979.0,
      980.2,
      1161.5,
      1162.9,
      1252.3,
      1391.9
     ],
     "tpot_quantiles_ms": [
      63.5,
      64.7,
      65.4,
      65.5,
      65.8,
      66.0,
      66.3,
      66.6,
      66.7,
      66.8,
      67.0,
      67.0,
      67.1,
      67.1,
      67.2,
      67.3,
      67.3,
      67.5,
      67.8,
      68.9,
      69.3,
      70.3,
      70.9,
      71.7,
      72.4,
      73.0,
      73.1,
      73.5,
      74.0,
      74.7,
      75.0,
      75.7,
      76.4,
      77.2,
      77.4,
      77.7,
      78.1,
      78.5,
      79.0,
      79.2,
      80.0,
      80.5,
      81.1,
      81.8,
      82.5,
      83.3,
      84.9,
      88.6,
      92.6,
      101.8,
      215.7
     ]
    },
    {
     "x": 0.5,
     "output_throughput": {
      "median": 169.6,
      "min": 165.18,
      "max": 173.44,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 352.6,
      "min": 347.59,
      "max": 376.59,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1796.9,
      "min": 1682.52,
      "max": 1850.83,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 81.41,
      "min": 81.01,
      "max": 82.48,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 88.78,
      "min": 87.63,
      "max": 90.05,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 190.72,
      "min": 189.37,
      "max": 218.99,
      "n": 3
     },
     "median_itl_ms": {
      "median": 67.78,
      "min": 67.75,
      "max": 67.79,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.46,
      "min": 1.44,
      "max": 1.5,
      "n": 3
     },
     "request_throughput": {
      "median": 1.75,
      "min": 1.74,
      "max": 1.76,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1455.0,
       "min": 1425.0,
       "max": 1455.0,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      95.5,
      267.7,
      271.8,
      274.0,
      275.6,
      277.2,
      278.3,
      279.0,
      280.3,
      281.7,
      283.0,
      283.5,
      290.5,
      302.0,
      305.9,
      310.1,
      313.1,
      313.9,
      315.9,
      317.0,
      317.9,
      319.3,
      321.8,
      322.3,
      338.9,
      350.9,
      362.1,
      378.7,
      381.9,
      391.6,
      401.7,
      412.1,
      420.5,
      425.4,
      430.9,
      434.4,
      443.3,
      460.2,
      561.7,
      650.6,
      651.2,
      651.7,
      703.2,
      1242.9,
      1244.6,
      1252.9,
      1254.2,
      1682.7,
      1796.4,
      1813.4,
      1912.8
     ],
     "tpot_quantiles_ms": [
      64.6,
      66.7,
      67.6,
      69.7,
      71.0,
      72.2,
      72.7,
      73.5,
      74.6,
      75.6,
      76.6,
      77.1,
      77.3,
      77.7,
      77.9,
      78.4,
      78.8,
      79.0,
      79.3,
      79.6,
      79.9,
      80.3,
      80.7,
      80.9,
      81.2,
      81.5,
      81.7,
      81.9,
      82.4,
      82.6,
      82.7,
      82.8,
      83.1,
      83.4,
      83.7,
      83.9,
      84.0,
      84.0,
      84.1,
      84.2,
      84.4,
      84.7,
      84.9,
      85.2,
      85.5,
      85.7,
      85.9,
      86.5,
      87.2,
      89.9,
      90.6
     ]
    },
    {
     "x": 1.0,
     "output_throughput": {
      "median": 181.71,
      "min": 181.68,
      "max": 182.82,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 1499.34,
      "min": 1460.33,
      "max": 1532.05,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2315.72,
      "min": 2304.78,
      "max": 2392.47,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 76.76,
      "min": 75.85,
      "max": 77.16,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 86.18,
      "min": 85.62,
      "max": 86.19,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 502.21,
      "min": 500.1,
      "max": 503.08,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.65,
      "min": 69.56,
      "max": 69.69,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.35,
      "min": 0.33,
      "max": 0.38,
      "n": 3
     },
     "request_throughput": {
      "median": 1.42,
      "min": 1.42,
      "max": 1.43,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1410.0,
       "min": 1410.0,
       "max": 1410.0,
       "n": 3
      },
      "max_temp_c": 85.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      96.0,
      347.4,
      354.6,
      358.7,
      629.3,
      634.0,
      652.9,
      654.9,
      775.5,
      786.6,
      788.9,
      870.0,
      982.7,
      1054.3,
      1056.5,
      1212.7,
      1215.8,
      1244.7,
      1245.7,
      1285.6,
      1298.0,
      1311.1,
      1350.0,
      1397.7,
      1436.3,
      1462.3,
      1569.1,
      1580.3,
      1588.3,
      1590.6,
      1594.8,
      1601.6,
      1602.2,
      1604.5,
      1614.2,
      1637.1,
      1645.1,
      1682.4,
      1685.0,
      1703.0,
      1728.4,
      1798.0,
      1813.7,
      1815.4,
      1876.1,
      1916.6,
      2104.5,
      2186.0,
      2304.4,
      2315.3,
      2392.7
     ],
     "tpot_quantiles_ms": [
      69.5,
      69.6,
      70.7,
      72.2,
      73.2,
      73.9,
      74.0,
      74.0,
      74.0,
      74.0,
      74.1,
      74.7,
      74.8,
      74.8,
      74.9,
      75.0,
      75.0,
      75.4,
      75.6,
      76.0,
      76.1,
      76.1,
      76.3,
      76.4,
      76.4,
      76.7,
      76.8,
      76.8,
      77.2,
      77.3,
      77.5,
      77.9,
      78.3,
      78.6,
      79.0,
      79.2,
      79.9,
      80.6,
      80.7,
      80.8,
      80.9,
      80.9,
      82.4,
      82.6,
      82.6,
      83.2,
      83.7,
      84.1,
      84.9,
      85.9,
      86.8
     ]
    }
   ]
  },
  "sglang": {
   "kv_pool_tokens": 24094,
   "points": [
    {
     "x": 0.0,
     "output_throughput": {
      "median": 158.18,
      "min": 146.7,
      "max": 176.92,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 283.34,
      "min": 264.59,
      "max": 290.99,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1134.48,
      "min": 946.31,
      "max": 1369.77,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 86.71,
      "min": 78.14,
      "max": 90.6,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 119.93,
      "min": 103.64,
      "max": 145.11,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 301.93,
      "min": 287.18,
      "max": 304.9,
      "n": 3
     },
     "median_itl_ms": {
      "median": 67.22,
      "min": 66.96,
      "max": 67.42,
      "n": 3
     },
     "goodput_rps": {
      "median": 2.2,
      "min": 1.94,
      "max": 2.33,
      "n": 3
     },
     "request_throughput": {
      "median": 2.41,
      "min": 2.22,
      "max": 2.52,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1485.0,
       "min": 1485.0,
       "max": 1500.0,
       "n": 3
      },
      "max_temp_c": 81.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      92.2,
      160.8,
      162.3,
      176.3,
      187.3,
      190.8,
      191.2,
      191.8,
      192.0,
      192.7,
      196.3,
      200.7,
      202.0,
      203.6,
      205.5,
      206.8,
      217.3,
      221.9,
      234.7,
      259.1,
      261.8,
      265.5,
      266.6,
      269.4,
      271.0,
      272.0,
      273.6,
      284.1,
      293.0,
      297.9,
      299.7,
      302.7,
      307.9,
      314.4,
      323.3,
      338.0,
      365.1,
      390.2,
      453.1,
      713.0,
      714.2,
      715.6,
      778.6,
      780.0,
      873.5,
      874.4,
      876.0,
      945.8,
      1134.0,
      1368.7,
      1370.0
     ],
     "tpot_quantiles_ms": [
      64.5,
      66.4,
      66.9,
      67.4,
      68.4,
      69.5,
      69.9,
      72.0,
      72.4,
      73.5,
      74.2,
      74.5,
      75.3,
      76.3,
      77.0,
      77.7,
      78.0,
      78.2,
      78.5,
      78.8,
      80.1,
      80.4,
      80.8,
      81.5,
      82.3,
      84.1,
      84.7,
      84.9,
      86.3,
      87.1,
      88.1,
      88.4,
      88.7,
      88.9,
      89.5,
      90.4,
      90.7,
      91.0,
      91.4,
      91.9,
      92.1,
      92.3,
      92.9,
      93.5,
      94.3,
      95.4,
      96.4,
      97.4,
      99.5,
      110.8,
      186.5
     ]
    },
    {
     "x": 0.5,
     "output_throughput": {
      "median": 159.55,
      "min": 155.75,
      "max": 162.03,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 325.06,
      "min": 321.49,
      "max": 342.17,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 1714.26,
      "min": 1711.37,
      "max": 1865.8,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 87.78,
      "min": 87.58,
      "max": 87.81,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 96.34,
      "min": 95.17,
      "max": 97.74,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 362.58,
      "min": 360.23,
      "max": 370.02,
      "n": 3
     },
     "median_itl_ms": {
      "median": 68.35,
      "min": 68.33,
      "max": 68.39,
      "n": 3
     },
     "goodput_rps": {
      "median": 1.25,
      "min": 1.22,
      "max": 1.27,
      "n": 3
     },
     "request_throughput": {
      "median": 1.64,
      "min": 1.63,
      "max": 1.65,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1470.0,
       "min": 1432.5,
       "max": 1477.5,
       "n": 3
      },
      "max_temp_c": 83.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      93.1,
      194.5,
      260.5,
      263.9,
      267.4,
      268.6,
      269.6,
      270.8,
      274.2,
      275.6,
      276.7,
      284.6,
      290.4,
      293.1,
      299.1,
      300.7,
      303.8,
      304.7,
      305.4,
      307.3,
      308.4,
      310.3,
      310.6,
      311.8,
      316.5,
      326.1,
      335.7,
      345.4,
      350.4,
      360.4,
      364.3,
      371.9,
      380.8,
      390.0,
      402.1,
      417.8,
      430.1,
      471.8,
      567.2,
      1007.3,
      1008.5,
      1010.0,
      1011.6,
      1166.9,
      1168.1,
      1169.2,
      1710.6,
      1712.5,
      1713.7,
      1864.7,
      1866.1
     ],
     "tpot_quantiles_ms": [
      65.1,
      67.7,
      68.2,
      71.3,
      72.2,
      74.2,
      75.9,
      77.1,
      77.7,
      80.1,
      80.9,
      82.7,
      83.1,
      83.4,
      83.9,
      84.3,
      84.9,
      85.3,
      85.8,
      86.2,
      86.7,
      86.9,
      87.0,
      87.2,
      87.3,
      87.6,
      88.1,
      88.9,
      89.0,
      89.4,
      89.7,
      90.0,
      90.5,
      90.6,
      90.7,
      90.8,
      91.0,
      91.3,
      91.4,
      91.6,
      91.7,
      92.2,
      92.3,
      92.8,
      93.5,
      93.6,
      93.8,
      94.5,
      94.9,
      95.7,
      103.2
     ]
    },
    {
     "x": 1.0,
     "output_throughput": {
      "median": 182.28,
      "min": 181.49,
      "max": 182.7,
      "n": 3
     },
     "median_ttft_ms": {
      "median": 1775.03,
      "min": 1668.0,
      "max": 1830.12,
      "n": 3
     },
     "p99_ttft_ms": {
      "median": 2449.88,
      "min": 2420.12,
      "max": 2494.15,
      "n": 3
     },
     "median_tpot_ms": {
      "median": 74.65,
      "min": 74.32,
      "max": 75.52,
      "n": 3
     },
     "p99_tpot_ms": {
      "median": 87.57,
      "min": 87.08,
      "max": 87.65,
      "n": 3
     },
     "p99_itl_ms": {
      "median": 80.57,
      "min": 79.28,
      "max": 80.91,
      "n": 3
     },
     "median_itl_ms": {
      "median": 69.64,
      "min": 69.63,
      "max": 69.73,
      "n": 3
     },
     "goodput_rps": {
      "median": 0.36,
      "min": 0.22,
      "max": 0.45,
      "n": 3
     },
     "request_throughput": {
      "median": 1.42,
      "min": 1.42,
      "max": 1.43,
      "n": 3
     },
     "completed": [
      64,
      64,
      64
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 1440.0,
       "min": 1432.5,
       "max": 1470.0,
       "n": 3
      },
      "max_temp_c": 85.0,
      "thermal_share": 0.0
     },
     "ttft_quantiles_ms": [
      93.6,
      189.2,
      191.3,
      214.8,
      493.8,
      508.5,
      610.5,
      613.2,
      623.5,
      630.9,
      757.8,
      920.8,
      921.8,
      1183.5,
      1642.0,
      1643.5,
      1658.9,
      1660.8,
      1662.5,
      1663.7,
      1671.1,
      1673.3,
      1761.1,
      1762.3,
      1785.7,
      1786.7,
      1787.4,
      1788.7,
      1789.5,
      1794.7,
      1797.0,
      1829.9,
      1831.6,
      1921.6,
      1922.9,
      2074.1,
      2075.4,
      2196.8,
      2315.1,
      2316.0,
      2339.4,
      2352.5,
      2361.8,
      2388.7,
      2408.5,
      2410.0,
      2420.2,
      2450.0,
      2468.3,
      2492.5,
      2494.3
     ],
     "tpot_quantiles_ms": [
      69.6,
      69.6,
      69.6,
      69.7,
      69.7,
      69.7,
      69.7,
      69.8,
      69.8,
      69.8,
      69.8,
      69.8,
      69.9,
      70.0,
      71.0,
      71.0,
      73.0,
      73.0,
      73.0,
      74.0,
      74.0,
      74.2,
      74.2,
      74.3,
      74.3,
      74.4,
      74.4,
      75.1,
      75.1,
      75.1,
      75.1,
      75.5,
      75.5,
      76.0,
      76.0,
      76.3,
      76.3,
      78.5,
      78.5,
      80.0,
      82.2,
      83.0,
      83.4,
      83.6,
      84.1,
      84.3,
      84.8,
      85.6,
      86.8,
      87.3,
      88.0
     ]
    }
   ]
  }
 }
}