{
  "lesson": "08-inference-metrics-goodput",
  "title": "Inference Metrics — TTFT, TPOT, ITL, Goodput, P99",
  "questions": [
    {
      "stage": "pre",
      "question": "Which components dominate TTFT (time to first token)?",
      "options": [
        "Disk I/O for weights",
        "Queue time, network request time, and prefill time",
        "Tokenizer GIL overhead",
        "Decode-only forward time"
      ],
      "correct": 1,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "Which metric does the lesson call the one that actually matters for product?",
      "options": [
        "Goodput — fraction of requests meeting every SLO constraint simultaneously",
        "Aggregate throughput in tokens per second",
        "Mean ITL",
        "GPU duty cycle"
      ],
      "correct": 0,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "Why is mean the wrong statistic to report for LLM latency?",
      "options": [
        "LLM latency distributions are right-skewed; users routinely hit P99 outliers that mean hides",
        "Mean is never computable on streaming responses",
        "Mean only works for prefill, not decode",
        "Mean is not supported by GenAI-Perf"
      ],
      "correct": 0,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "Why do GenAI-Perf and LLMPerf disagree on TPOT for the same run?",
      "options": [
        "They sample different requests",
        "GenAI-Perf only runs on Blackwell",
        "LLMPerf uses microseconds and GenAI-Perf uses milliseconds",
        "GenAI-Perf excludes TTFT from the ITL calculation; LLMPerf includes it, so tool choice changes the number"
      ],
      "correct": 3,
      "explanation": ""
    },
    {
      "stage": "post",
      "question": "For long-output requests (>500 tokens), which metric dominates end-to-end latency?",
      "options": [
        "Network response time",
        "TPOT times output length",
        "TTFT",
        "Cold-start time"
      ],
      "correct": 1,
      "explanation": ""
    },
    {
      "stage": "post",
      "question": "Which set best captures the lesson's reasonable consumer-facing SLO for a 70B chat model in 2026?",
      "options": [
        "P50 only, no percentiles above",
        "TTFT P99 800 ms, TPOT P99 25 ms, E2E P99 3 s for <300-token outputs, goodput >= 99%",
        "TTFT P99 8s, TPOT P99 200ms, goodput 50%",
        "Mean TTFT 10 ms, mean TPOT 1 ms"
      ],
      "correct": 1,
      "explanation": ""
    }
  ]
}