{
  "lesson": "01-text-processing",
  "title": "Text Processing — Tokenization, Stemming, Lemmatization",
  "questions": [
    {
      "stage": "pre",
      "question": "Why does a language model need preprocessing before training?",
      "options": [
        "Models read strings directly",
        "Preprocessing improves GPU utilization",
        "Models consume discrete integer tokens, not raw text",
        "Preprocessing is required only for languages without whitespace"
      ],
      "correct": 2,
      "explanation": "Models operate on integer token IDs; preprocessing bridges continuous language to discrete inputs."
    },
    {
      "stage": "pre",
      "question": "Which preprocessing operation is rule-based suffix stripping?",
      "options": [
        "POS tagging",
        "Stemming",
        "Lemmatization",
        "Tokenization"
      ],
      "correct": 1,
      "explanation": "Stemming chops suffixes with rules; it is fast but can be wrong."
    },
    {
      "stage": "check",
      "question": "What does the Porter stemmer step 1a return for the word 'ponies'?",
      "options": [
        "poni",
        "ponies",
        "pony",
        "ponie"
      ],
      "correct": 0,
      "explanation": "The 'ies' rule replaces the suffix with 'i', producing 'poni'; step 1b in real Porter cleans it up."
    },
    {
      "stage": "check",
      "question": "Why does lemmatization usually need a POS tag?",
      "options": [
        "Unicode handling",
        "Grammatical context disambiguates forms like 'better' (ADJ) versus 'better' (VERB)",
        "To reduce memory usage",
        "Speed"
      ],
      "correct": 1,
      "explanation": "Lemmas depend on grammatical role; without the tag the lookup is ambiguous."
    },
    {
      "stage": "check",
      "question": "When translating NLTK Penn Treebank POS tags to WordNet tags, what does a tag starting with 'V' map to?",
      "options": [
        "'r' (adverb)",
        "'a' (adjective)",
        "'v' (verb)",
        "'n' (noun)"
      ],
      "correct": 2,
      "explanation": "Verb-prefixed Penn Treebank tags map to WordNet 'v'."
    },
    {
      "stage": "post",
      "question": "What is 'training/inference mismatch' in NLP preprocessing?",
      "options": [
        "A GPU memory issue",
        "Training and serving apply different preprocessing, so the model sees an unfamiliar distribution at inference",
        "Mixing tokenizer libraries within a notebook",
        "Different batch sizes between train and test"
      ],
      "correct": 1,
      "explanation": "Different preprocessing at training vs inference is the most common production NLP failure."
    },
    {
      "stage": "post",
      "question": "Which tool would you reach for in a transformer pipeline instead of classical preprocessing?",
      "options": [
        "NLTK word_tokenize",
        "The model's own tokenizer (e.g. via tokenizers / transformers)",
        "spaCy en_core_web_sm",
        "A regex tokenizer"
      ],
      "correct": 1,
      "explanation": "Transformer models ship a paired tokenizer; classical preprocessing is bypassed."
    },
    {
      "stage": "post",
      "question": "Why pin NLTK and spaCy versions in requirements?",
      "options": [
        "Newer versions are slower",
        "Older versions support more languages",
        "Tokenizer and lemmatizer behavior shifts between minor releases, silently changing training distribution",
        "License compatibility"
      ],
      "correct": 2,
      "explanation": "Library upgrades can change tokenization output, drifting from the training distribution."
    }
  ]
}