{
 "profile": "l4-gpu1",
 "slo": {
  "ttft_ms": 1000.0,
  "tpot_ms": 100.0
 },
 "environment": null,
 "quantiles": [
  0.0,
  0.02,
  0.04,
  0.06,
  0.08,
  0.1,
  0.12,
  0.14,
  0.16,
  0.18,
  0.2,
  0.22,
  0.24,
  0.26,
  0.28,
  0.3,
  0.32,
  0.34,
  0.36,
  0.38,
  0.4,
  0.42,
  0.44,
  0.46,
  0.48,
  0.5,
  0.52,
  0.54,
  0.56,
  0.58,
  0.6,
  0.62,
  0.64,
  0.66,
  0.68,
  0.7,
  0.72,
  0.74,
  0.76,
  0.78,
  0.8,
  0.82,
  0.84,
  0.86,
  0.88,
  0.9,
  0.92,
  0.94,
  0.96,
  0.98,
  1.0
 ],
 "load": {
  "vllm-fast-b": {
   "kv_pool_tokens": 2928,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 3.98,
      "min": 3.94,
      "max": 4.01,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 1437.17,
      "min": 1432.09,
      "max": 1442.26,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 1536.41,
      "min": 1456.27,
      "max": 1616.55,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 241.7,
      "min": 240.36,
      "max": 243.03,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 250.83,
      "min": 244.78,
      "max": 256.88,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 274.22,
      "min": 255.01,
      "max": 293.44,
      "n": 2
     },
     "median_itl_ms": {
      "median": 241.0,
      "min": 240.67,
      "max": 241.32,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 2
     },
     "request_throughput": {
      "median": 0.03,
      "min": 0.03,
      "max": 0.03,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 375.0,
       "min": 375.0,
       "max": 375.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 1.0
     },
     "ttft_quantiles_ms": [
      277.9,
      277.9,
      280.4,
      280.4,
      1138.2,
      1223.6,
      1223.6,
      1309.9,
      1309.9,
      1379.9,
      1391.1,
      1391.1,
      1410.9,
      1411.5,
      1411.5,
      1413.8,
      1413.8,
      1415.4,
      1417.8,
      1417.8,
      1422.0,
      1429.0,
      1429.0,
      1435.2,
      1435.2,
      1437.2,
      1438.9,
      1438.9,
      1439.8,
      1439.8,
      1440.6,
      1443.1,
      1443.1,
      1443.9,
      1444.7,
      1444.7,
      1449.9,
      1449.9,
      1452.2,
      1453.1,
      1453.1,
      1456.8,
      1467.5,
      1467.5,
      1485.1,
      1485.1,
      1492.0,
      1543.8,
      1543.8,
      1605.6,
      1618.5
     ],
     "tpot_quantiles_ms": [
      238.0,
      238.0,
      238.5,
      238.5,
      239.2,
      239.3,
      239.3,
      239.6,
      239.6,
      239.8,
      239.9,
      239.9,
      240.0,
      240.1,
      240.1,
      240.2,
      240.2,
      240.3,
      240.3,
      240.3,
      240.4,
      240.7,
      240.7,
      240.8,
      240.8,
      240.8,
      241.0,
      241.0,
      241.7,
      241.7,
      241.8,
      242.3,
      242.3,
      242.3,
      242.3,
      242.3,
      242.8,
      242.8,
      243.3,
      243.4,
      243.4,
      245.2,
      246.6,
      246.6,
      246.9,
      246.9,
      247.6,
      253.2,
      253.2,
      253.6,
      257.5
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 13.02,
      "min": 12.97,
      "max": 13.06,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 4944.93,
      "min": 4764.24,
      "max": 5125.62,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 5342.51,
      "min": 5265.54,
      "max": 5419.48,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 272.6,
      "min": 270.11,
      "max": 275.08,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 303.82,
      "min": 302.11,
      "max": 305.54,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 290.42,
      "min": 286.11,
      "max": 294.73,
      "n": 2
     },
     "median_itl_ms": {
      "median": 271.87,
      "min": 271.19,
      "max": 272.56,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 2
     },
     "request_throughput": {
      "median": 0.1,
      "min": 0.1,
      "max": 0.1,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 427.5,
       "min": 420.0,
       "max": 435.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 1.0
     },
     "ttft_quantiles_ms": [
      276.5,
      276.5,
      287.4,
      287.4,
      1503.1,
      1558.9,
      1558.9,
      1560.4,
      1560.4,
      1674.5,
      1690.9,
      1690.9,
      1729.6,
      3997.6,
      3997.6,
      3998.2,
      3998.2,
      3998.6,
      4143.7,
      4143.7,
      4144.3,
      4144.7,
      4144.7,
      4763.9,
      4763.9,
      4764.6,
      4765.5,
      4765.5,
      5125.4,
      5125.4,
      5125.8,
      5127.2,
      5127.2,
      5257.8,
      5258.3,
      5258.3,
      5259.6,
      5259.6,
      5264.0,
      5264.6,
      5264.6,
      5265.7,
      5380.6,
      5380.6,
      5381.1,
      5381.1,
      5382.7,
      5417.8,
      5417.8,
      5418.8,
      5419.6
     ],
     "tpot_quantiles_ms": [
      266.9,
      266.9,
      266.9,
      266.9,
      266.9,
      267.6,
      267.6,
      267.6,
      267.6,
      267.6,
      268.1,
      268.1,
      268.1,
      268.1,
      268.1,
      268.2,
      268.2,
      268.2,
      268.2,
      268.2,
      270.1,
      270.1,
      270.1,
      270.1,
      270.1,
      274.8,
      274.8,
      274.8,
      274.8,
      274.8,
      275.1,
      275.1,
      275.1,
      275.1,
      276.8,
      276.8,
      276.8,
      276.8,
      276.8,
      293.3,
      293.3,
      295.1,
      296.2,
      296.2,
      296.6,
      296.6,
      297.2,
      303.0,
      303.0,
      305.4,
      305.6
     ]
    }
   ]
  },
  "vllm-ngram": {
   "kv_pool_tokens": 2928,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 7.39,
      "min": 7.3,
      "max": 7.49,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 1428.05,
      "min": 1420.77,
      "max": 1435.34,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 1659.74,
      "min": 1615.52,
      "max": 1703.96,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 76.58,
      "min": 72.51,
      "max": 80.66,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 256.21,
      "min": 248.39,
      "max": 264.03,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 284.06,
      "min": 276.87,
      "max": 291.26,
      "n": 2
     },
     "median_itl_ms": {
      "median": 265.39,
      "min": 265.32,
      "max": 265.47,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 2
     },
     "request_throughput": {
      "median": 0.06,
      "min": 0.06,
      "max": 0.06,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 412.5,
       "min": 405.0,
       "max": 420.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 1.0
     },
     "ttft_quantiles_ms": [
      270.9,
      270.9,
      272.5,
      272.5,
      1200.3,
      1202.3,
      1202.3,
      1202.5,
      1202.5,
      1212.6,
      1217.7,
      1217.7,
      1373.3,
      1395.4,
      1395.4,
      1403.5,
      1403.5,
      1414.1,
      1419.3,
      1419.3,
      1419.7,
      1427.4,
      1427.4,
      1428.6,
      1428.6,
      1431.4,
      1435.1,
      1435.1,
      1435.6,
      1435.6,
      1435.8,
      1436.2,
      1436.2,
      1436.7,
      1436.8,
      1436.8,
      1445.6,
      1445.6,
      1451.6,
      1504.1,
      1504.1,
      1504.5,
      1511.6,
      1511.6,
      1517.6,
      1517.6,
      1526.4,
      1551.5,
      1551.5,
      1626.8,
      1748.5
     ],
     "tpot_quantiles_ms": [
      257.6,
      257.6,
      257.7,
      257.7,
      259.2,
      260.4,
      260.4,
      261.0,
      261.0,
      262.2,
      262.8,
      262.8,
      264.9,
      265.7,
      265.7,
      266.9,
      266.9,
      267.0,
      267.3,
      267.3,
      267.8,
      268.2,
      268.2,
      268.5,
      268.5,
      268.6,
      269.5,
      269.5,
      270.5,
      270.5,
      270.7,
      270.8,
      270.8,
      270.9,
      271.0,
      271.0,
      271.0,
      271.0,
      271.1,
      271.4,
      271.4,
      271.8,
      271.8,
      271.8,
      272.0,
      272.0,
      272.8,
      276.3,
      276.3,
      277.4,
      278.5
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 19.22,
      "min": 18.75,
      "max": 19.69,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 1924.53,
      "min": 1858.69,
      "max": 1990.37,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 3859.52,
      "min": 3818.24,
      "max": 3900.8,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 101.25,
      "min": 95.01,
      "max": 107.49,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 349.44,
      "min": 347.63,
      "max": 351.25,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 1691.13,
      "min": 1668.81,
      "max": 1713.46,
      "n": 2
     },
     "median_itl_ms": {
      "median": 283.7,
      "min": 282.9,
      "max": 284.51,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 2
     },
     "request_throughput": {
      "median": 0.15,
      "min": 0.15,
      "max": 0.15,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 442.5,
       "min": 435.0,
       "max": 450.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 1.0
     },
     "ttft_quantiles_ms": [
      264.1,
      264.1,
      275.3,
      275.3,
      1673.4,
      1706.4,
      1706.4,
      1733.2,
      1733.2,
      1746.4,
      1746.7,
      1746.7,
      1749.2,
      1750.1,
      1750.1,
      1754.2,
      1754.2,
      1762.5,
      1864.2,
      1864.2,
      1894.3,
      1932.5,
      1932.5,
      1954.9,
      1954.9,
      1973.6,
      1982.8,
      1982.8,
      1998.0,
      1998.0,
      1998.8,
      2048.3,
      2048.3,
      2149.8,
      2158.1,
      2158.1,
      2777.3,
      2777.3,
      2858.5,
      2970.3,
      2970.3,
      3729.2,
      3817.4,
      3817.4,
      3817.9,
      3817.9,
      3818.3,
      3900.2,
      3900.2,
      3900.8,
      3900.8
     ],
     "tpot_quantiles_ms": [
      263.8,
      263.8,
      269.8,
      269.8,
      278.0,
      281.9,
      281.9,
      294.1,
      294.1,
      295.7,
      301.5,
      301.5,
      308.8,
      316.2,
      316.2,
      319.3,
      319.3,
      320.7,
      326.9,
      326.9,
      333.2,
      333.5,
      333.5,
      334.0,
      334.0,
      335.8,
      341.1,
      341.1,
      341.5,
      341.5,
      342.7,
      345.5,
      345.5,
      348.6,
      352.3,
      352.3,
      357.2,
      357.2,
      357.4,
      358.1,
      358.1,
      361.2,
      368.2,
      368.2,
      368.2,
      368.2,
      369.8,
      380.6,
      380.6,
      386.7,
      393.7
     ]
    }
   ]
  },
  "vllm-no-async": {
   "kv_pool_tokens": 2928,
   "points": [
    {
     "x": 1.0,
     "output_throughput": {
      "median": 3.87,
      "min": 3.8,
      "max": 3.93,
      "n": 2
     },
     "median_ttft_ms": {
      "median": 1466.72,
      "min": 1430.97,
      "max": 1502.47,
      "n": 2
     },
     "p99_ttft_ms": {
      "median": 1783.18,
      "min": 1699.69,
      "max": 1866.67,
      "n": 2
     },
     "median_tpot_ms": {
      "median": 249.58,
      "min": 245.33,
      "max": 253.83,
      "n": 2
     },
     "p99_tpot_ms": {
      "median": 259.08,
      "min": 249.41,
      "max": 268.74,
      "n": 2
     },
     "p99_itl_ms": {
      "median": 282.54,
      "min": 257.85,
      "max": 307.24,
      "n": 2
     },
     "median_itl_ms": {
      "median": 249.77,
      "min": 245.75,
      "max": 253.79,
      "n": 2
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 2
     },
     "request_throughput": {
      "median": 0.03,
      "min": 0.03,
      "max": 0.03,
      "n": 2
     },
     "completed": [
      16,
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 367.5,
       "min": 360.0,
       "max": 375.0,
       "n": 2
      },
      "max_temp_c": 88.0,
      "thermal_share": 1.0
     },
     "ttft_quantiles_ms": [
      279.9,
      279.9,
      287.2,
      287.2,
      1192.8,
      1260.3,
      1260.3,
      1301.6,
      1301.6,
      1321.3,
      1406.9,
      1406.9,
      1418.0,
      1419.7,
      1419.7,
      1427.2,
      1427.2,
      1428.2,
      1433.8,
      1433.8,
      1435.6,
      1438.9,
      1438.9,
      1448.9,
      1448.9,
      1451.3,
      1451.4,
      1451.4,
      1458.3,
      1458.3,
      1470.6,
      1478.3,
      1478.3,
      1483.0,
      1491.8,
      1491.8,
      1499.4,
      1499.4,
      1505.5,
      1512.5,
      1512.5,
      1517.3,
      1529.1,
      1529.1,
      1652.6,
      1652.6,
      1742.3,
      1751.4,
      1751.4,
      1787.1,
      1880.7
     ],
     "tpot_quantiles_ms": [
      239.3,
      239.3,
      240.6,
      240.6,
      240.8,
      243.5,
      243.5,
      243.7,
      243.7,
      243.7,
      243.9,
      243.9,
      243.9,
      244.2,
      244.2,
      244.4,
      244.4,
      245.2,
      245.4,
      245.4,
      245.5,
      246.5,
      246.5,
      247.3,
      247.3,
      248.0,
      248.6,
      248.6,
      249.3,
      249.3,
      249.3,
      249.4,
      249.4,
      251.6,
      251.9,
      251.9,
      252.2,
      252.2,
      252.8,
      254.8,
      254.8,
      255.3,
      255.7,
      255.7,
      255.8,
      255.8,
      259.4,
      262.6,
      262.6,
      264.3,
      269.5
     ]
    },
    {
     "x": 4.0,
     "output_throughput": {
      "median": 12.85,
      "min": 12.85,
      "max": 12.85,
      "n": 1
     },
     "median_ttft_ms": {
      "median": 4690.01,
      "min": 4690.01,
      "max": 4690.01,
      "n": 1
     },
     "p99_ttft_ms": {
      "median": 5302.54,
      "min": 5302.54,
      "max": 5302.54,
      "n": 1
     },
     "median_tpot_ms": {
      "median": 277.22,
      "min": 277.22,
      "max": 277.22,
      "n": 1
     },
     "p99_tpot_ms": {
      "median": 306.43,
      "min": 306.43,
      "max": 306.43,
      "n": 1
     },
     "p99_itl_ms": {
      "median": 309.49,
      "min": 309.49,
      "max": 309.49,
      "n": 1
     },
     "median_itl_ms": {
      "median": 276.84,
      "min": 276.84,
      "max": 276.84,
      "n": 1
     },
     "goodput_rps": {
      "median": 0.0,
      "min": 0.0,
      "max": 0.0,
      "n": 1
     },
     "request_throughput": {
      "median": 0.1,
      "min": 0.1,
      "max": 0.1,
      "n": 1
     },
     "completed": [
      16
     ],
     "gpu": {
      "median_sm_mhz": {
       "median": 450.0,
       "min": 450.0,
       "max": 450.0,
       "n": 1
      },
      "max_temp_c": 88.0,
      "thermal_share": 1.0
     },
     "ttft_quantiles_ms": [
      288.0,
      288.0,
      288.0,
      288.0,
      1435.6,
      1435.6,
      1435.6,
      1495.2,
      1495.2,
      1495.2,
      1577.5,
      1577.5,
      1577.5,
      1577.5,
      3903.2,
      3903.2,
      3903.2,
      3903.8,
      3903.8,
      3903.8,
      3904.2,
      3904.2,
      3904.2,
      3904.2,
      4689.7,
      4689.7,
      4689.7,
      4690.4,
      4690.4,
      4690.4,
      4691.4,
      4691.4,
      4691.4,
      4691.4,
      5206.1,
      5206.1,
      5206.1,
      5206.5,
      5206.5,
      5206.5,
      5207.8,
      5207.8,
      5207.8,
      5207.8,
      5301.0,
      5301.0,
      5301.0,
      5301.4,
      5301.4,
      5301.4,
      5302.7
     ],
     "tpot_quantiles_ms": [
      271.9,
      271.9,
      271.9,
      271.9,
      271.9,
      271.9,
      271.9,
      271.9,
      271.9,
      271.9,
      275.8,
      275.8,
      275.8,
      275.8,
      275.8,
      275.8,
      275.8,
      275.8,
      275.8,
      275.8,
      277.2,
      277.2,
      277.2,
      277.2,
      277.2,
      277.2,
      277.2,
      277.2,
      277.2,
      277.2,
      279.8,
      279.8,
      279.8,
      279.8,
      279.8,
      279.8,
      279.8,
      279.8,
      279.8,
      279.8,
      298.3,
      298.3,
      298.3,
      298.3,
      305.0,
      305.0,
      305.0,
      305.4,
      305.4,
      305.4,
      306.6
     ]
    }
   ]
  }
 }
}