{
  "lesson": "02-bag-of-words-tfidf",
  "title": "Bag of Words, TF-IDF, and Text Representation",
  "questions": [
    {
      "stage": "pre",
      "question": "What does Bag of Words throw away?",
      "options": [
        "Punctuation",
        "Token order",
        "Vocabulary size",
        "Document length"
      ],
      "correct": 1,
      "explanation": "BoW counts tokens per document but discards their sequence."
    },
    {
      "stage": "pre",
      "question": "Why scale TF by an IDF factor?",
      "options": [
        "To normalize document length",
        "Words that appear in every document carry little discriminative signal and should be downweighted",
        "To remove punctuation",
        "To accelerate training"
      ],
      "correct": 1,
      "explanation": "IDF penalizes ubiquitous words and boosts rare ones."
    },
    {
      "stage": "check",
      "question": "In the smoothed IDF formula log((N+1)/(df+1)) + 1, what does the trailing +1 ensure?",
      "options": [
        "Numerical stability for large N",
        "Compatibility with raw counts",
        "A word that appears in every document still has IDF 1 instead of 0",
        "Faster computation"
      ],
      "correct": 2,
      "explanation": "The +1 keeps ubiquitous words at IDF=1 so they are not zeroed out, matching scikit-learn's default."
    },
    {
      "stage": "check",
      "question": "Why L2-normalize TF-IDF rows before cosine similarity?",
      "options": [
        "To convert sparse vectors to dense",
        "Longer documents would otherwise dominate similarity scores; normalization puts all docs on the unit hypersphere",
        "To remove zero entries",
        "To compress the vocabulary"
      ],
      "correct": 1,
      "explanation": "L2 normalization removes document-length bias and turns cosine similarity into a dot product."
    },
    {
      "stage": "check",
      "question": "Which TfidfVectorizer setting is risky to enable for sentiment analysis?",
      "options": [
        "sublinear_tf=True",
        "ngram_range=(1, 2)",
        "min_df=2",
        "stop_words='english'"
      ],
      "correct": 3,
      "explanation": "English stopword lists drop negations like 'not', which carry sentiment signal."
    },
    {
      "stage": "post",
      "question": "Which task does TF-IDF still win in 2026?",
      "options": [
        "Machine translation",
        "Open-ended dialogue",
        "Image captioning",
        "Spam detection, log anomaly flagging, and low-latency narrow classification"
      ],
      "correct": 3,
      "explanation": "TF-IDF beats embeddings when word presence is the signal and explainability or speed matter."
    },
    {
      "stage": "post",
      "question": "Why does TF-IDF fail on the pair 'The movie was not good' vs 'The movie was excellent'?",
      "options": [
        "TF-IDF requires bigrams to work",
        "Both documents share most tokens; bag-of-words has no notion of negation or word order",
        "TF-IDF cannot handle stopwords",
        "Embeddings overlap is too high"
      ],
      "correct": 1,
      "explanation": "Without word order or syntactic context, BoW cannot model that 'not' flips the sentiment of 'good'."
    },
    {
      "stage": "post",
      "question": "What is the TF-IDF weighted embedding hybrid?",
      "options": [
        "Concatenating BoW and dense embeddings",
        "Running PCA on TF-IDF then re-embedding",
        "Using TF-IDF weights as a pooling weight over per-token embeddings before averaging",
        "Training BERT on TF-IDF features"
      ],
      "correct": 2,
      "explanation": "The hybrid weights each token's embedding by its TF-IDF score and averages, blending semantic capacity with rare-word emphasis."
    }
  ]
}