{
  "lesson": "19-benchmarks-swebench-gaia",
  "title": "Benchmarks: SWE-bench, GAIA, AgentBench",
  "questions": [
    {
      "stage": "pre",
      "question": "What does SWE-bench's evaluator check on a candidate patch?",
      "options": [
        "Patch length under 200 lines",
        "Previously failing tests now pass (FAIL_TO_PASS) and previously passing tests still pass (PASS_TO_PASS)",
        "BLEU score against the reference fix",
        "Patch passes a separate LLM judge"
      ],
      "correct": 1,
      "explanation": "The harness gates on test transitions: bug-revealing tests must flip while regression tests must stay green."
    },
    {
      "stage": "pre",
      "question": "Why does SWE-bench Verified exist?",
      "options": [
        "It includes more languages",
        "It runs faster",
        "OpenAI's 500-task human-curated subset removes ambiguous issues and unreliable tests",
        "It uses a different patch format"
      ],
      "correct": 2,
      "explanation": "Verified is the cleaner subset for credible reporting."
    },
    {
      "stage": "check",
      "question": "What did SWE-bench+ find about successful patches?",
      "options": [
        "32.67% leaked solution text in the issue and 31.08% had suspiciously weak test coverage",
        "Patches always exceeded 1000 lines",
        "There is no contamination",
        "All patches were memorized"
      ],
      "correct": 0,
      "explanation": "SWE-bench+ flagged solution leakage and weak coverage on a large fraction of successful patches."
    },
    {
      "stage": "check",
      "question": "What is GAIA's design philosophy?",
      "options": [
        "Hard for humans, easy for AI",
        "Pure benchmark of vector retrieval",
        "Only single-turn questions",
        "Conceptually simple for humans (about 92%) but hard for AI (early GPT-4 with plugins: about 15%)"
      ],
      "correct": 3,
      "explanation": "GAIA is intentionally easy-for-humans, hard-for-AI, testing reasoning + tools + modality."
    },
    {
      "stage": "check",
      "question": "Which is NOT one of AgentBench's environment categories?",
      "options": [
        "Games (Alfworld, LTP)",
        "Web (WebShop, Mind2Web)",
        "Gradient (RL, IRL)",
        "Code (Bash, DB, KG)"
      ],
      "correct": 2,
      "explanation": "AgentBench covers code, games, web, and open-ended generation. There is no gradient category."
    },
    {
      "stage": "post",
      "question": "What does the lesson identify as the wrong way to report SWE-bench numbers?",
      "options": [
        "Reporting per-repo breakdowns",
        "Reporting one aggregate number without mentioning Verified or SWE-bench+ context",
        "Reporting step counts",
        "Reporting wall-clock"
      ],
      "correct": 1,
      "explanation": "Single-number fixation hides contamination and cost; always report Verified and per-distribution context."
    },
    {
      "stage": "post",
      "question": "Which dimension do these benchmarks NOT measure?",
      "options": [
        "Test transitions",
        "Step counts",
        "Per-task success",
        "Real-world operational cost (tokens, wall-clock), adversarial safety, and your own domain"
      ],
      "correct": 3,
      "explanation": "Benchmarks aggregate; they do not capture cost, adversarial robustness, or your domain."
    }
  ]
}