{
  "lesson": "05-constitutional-ai-rlaif",
  "title": "Constitutional AI and RLAIF",
  "questions": [
    {
      "stage": "pre",
      "question": "What is the core substitution in Constitutional AI versus standard RLHF?",
      "options": [
        "It removes the reward model entirely",
        "It uses synthetic prompts instead of real ones",
        "It replaces the human preference labeler with an AI labeler that reads a list of principles",
        "It replaces PPO with DPO"
      ],
      "correct": 2,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "What does the first (SFT) phase of Constitutional AI do?",
      "options": [
        "Generates synthetic principles by clustering labeler comments",
        "Produces an initial response, critiques it under a sampled constitution principle, then revises and uses the revision as SFT target",
        "Performs PPO with a constitution-derived reward",
        "Trains a reward model on AI-generated preferences only"
      ],
      "correct": 1,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "What does the second (RLAIF) phase do?",
      "options": [
        "Skips PPO and applies a single supervised loss",
        "Trains a reward model on AI-generated preferences from a feedback model and runs PPO against it",
        "Has humans re-label the SFT outputs",
        "Distills the constitution into the tokenizer"
      ],
      "correct": 1,
      "explanation": ""
    },
    {
      "stage": "check",
      "question": "What is the tier order in Anthropic's January 2026 four-tier Claude constitution?",
      "options": [
        "Helpful > broadly ethical > platform rules > avoid catastrophe",
        "Avoid catastrophic outcomes > follow Anthropic guidelines > broadly ethical > helpful and candid",
        "Broadly ethical > helpful > platform rules > avoid catastrophe",
        "Platform rules > avoid catastrophe > helpful > broadly ethical"
      ],
      "correct": 1,
      "explanation": ""
    },
    {
      "stage": "post",
      "question": "Which failure mode is NOT necessarily fixed by switching from RLHF to RLAIF?",
      "options": [
        "Labeler scarcity",
        "Labeler cost",
        "Reward hacking / Goodhart's Law",
        "Labeler psychological inconsistency"
      ],
      "correct": 2,
      "explanation": ""
    },
    {
      "stage": "post",
      "question": "What is the headline shift between Constitutional Classifiers v1 and v2 (2026)?",
      "options": [
        "v2 increased compute overhead from 1% to 23.7%",
        "v2 removed the classifier and relied on constitution alone",
        "v2 dropped compute overhead from 23.7% to about 1% with low successful attack rate",
        "v2 only runs on input, not output"
      ],
      "correct": 2,
      "explanation": ""
    }
  ]
}