{
  "lesson": "41-workbench-for-real-repos",
  "title": "The Workbench on a Real Repo",
  "questions": [
    {
      "stage": "pre",
      "question": "What is the goal of running the same task through prompt-only and workbench-guided pipelines?",
      "options": [
        "To benchmark GPUs",
        "To compare models",
        "To produce a before/after report you can hand to a skeptic with numbers, not arguments",
        "To pick a vendor"
      ],
      "correct": 2,
      "explanation": "The numbers do the arguing; the case is made on a real-feeling task, not a toy."
    },
    {
      "stage": "pre",
      "question": "Which is NOT one of the five outcomes measured?",
      "options": [
        "files_outside_scope",
        "model_perplexity",
        "acceptance_met",
        "tests_actually_run"
      ],
      "correct": 1,
      "explanation": "The five are tests_actually_run, acceptance_met, files_outside_scope, handoff_quality, reviewer_total."
    },
    {
      "stage": "check",
      "question": "What did LangChain's Anatomy of an Agent Harness measure on Terminal Bench 2.0?",
      "options": [
        "Top model lost 25 places",
        "Models all converged at top-3",
        "Harness changes did not move the rank",
        "Same model moved from outside top 30 to rank five by changing only the harness"
      ],
      "correct": 3,
      "explanation": "Twenty-five-rank delta on the same model is the headline harness-vs-model receipt."
    },
    {
      "stage": "check",
      "question": "What does the preprints.org paper cite as the failure rate for enterprise agent projects?",
      "options": [
        "8%",
        "About 88% fail to reach production, with failures clustering around runtime, not reasoning",
        "About 50%",
        "None fail"
      ],
      "correct": 1,
      "explanation": "The Harness Engineering for Language Agents preprint traces failures to runtime issues (stale state, brittle retries, overgrown context)."
    },
    {
      "stage": "check",
      "question": "What does WebAgent baseline accuracy do in long-context conditions?",
      "options": [
        "Stays flat",
        "Drops from 40-50% to under 10% mostly from infinite loops and goal loss",
        "Goes up by 30%",
        "Halves but stays above 30%"
      ],
      "correct": 1,
      "explanation": "Long-context collapse is what the Ralph Loop and handoff packet exist to absorb."
    },
    {
      "stage": "post",
      "question": "What does the lesson say about false negatives (cases where prompt-only is faster)?",
      "options": [
        "Single-step factual tasks, one-line lints, formatter runs are faster prompt-only; enumerate them honestly so the workbench is not framed as overkill",
        "They prove the workbench fails",
        "They do not exist",
        "They invalidate the harness thesis"
      ],
      "correct": 0,
      "explanation": "Honest enumeration of prompt-only-fastest cases keeps the harness argument credible."
    },
    {
      "stage": "post",
      "question": "Where do you cite the report from this lesson?",
      "options": [
        "Internal HR review",
        "Only at hackathons",
        "When someone wants to drop the verification gate 'just for this sprint', or when a new agent product launches and needs a portable time-savings benchmark",
        "Only in marketing decks"
      ],
      "correct": 2,
      "explanation": "The numbers travel further than the explanation; cite the report when pressure tries to short-circuit surfaces."
    }
  ]
}