{
  "lesson": "16-red-team-tooling-garak-llamaguard-pyrit",
  "title": "Red-Team Tooling - Garak, Llama Guard, PyRIT",
  "questions": [
    {
      "stage": "pre",
      "question": "What role does Llama Guard play in a typical 2026 deployment?",
      "options": [
        "A reasoning scaffolder for chain-of-thought",
        "A reward model for RLHF",
        "A tokenizer alternative",
        "An input and/or output safety classifier across MLCommons hazard categories, placed before and after the LLM"
      ],
      "correct": 3,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "Roughly how many MLCommons hazard categories does Llama Guard 3 cover?",
      "options": [
        "3",
        "40",
        "14",
        "7"
      ],
      "correct": 2,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "How is Garak's architecture organized?",
      "options": [
        "Probes (attack generators), Detectors (output scorers), Harnesses (campaign managers)",
        "Embeddings, retrievers, rerankers",
        "Tools, resources, prompts",
        "Routers, agents, queues"
      ],
      "correct": 0,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "What is PyRIT primarily designed for, and how does it differ from Garak?",
      "options": [
        "PyRIT runs deep multi-turn campaigns (converter chains, orchestrators like Crescendo and TAP, scoring) while Garak runs thousands of single-turn probes",
        "They are identical in scope and audience",
        "PyRIT is a Llama Guard variant; Garak is a Prompt-Guard variant",
        "PyRIT is a reward model trainer; Garak is a tokenizer"
      ],
      "correct": 0,
      "explanation": ""
    },
    {
      "stage": "post",
      "question": "What is the recommended 2026 default red-team stack configuration?",
      "options": [
        "Use PAIR instead of any tooling",
        "Run Garak once at launch only",
        "Replace LLM evaluation with manual review only",
        "Put Llama Guard on both sides of the model, run Garak nightly for regression, and run PyRIT for pre-release campaigns"
      ],
      "correct": 3,
      "explanation": ""
    },
    {
      "stage": "post",
      "question": "What evaluation pitfall is shared across Llama Guard, Garak, and PyRIT?",
      "options": [
        "They require white-box access",
        "They only work in English",
        "They cannot run on GPUs",
        "Reported attack/defense rates depend on the judge identity and probe staleness; you must specify the judge and refresh probes as models are patched"
      ],
      "correct": 3,
      "explanation": ""
    }
  ]
}