{
  "lesson": "69-end-to-end-rag-system",
  "title": "End-to-End RAG System",
  "questions": [
    {
      "stage": "pre",
      "question": "Why is the integration test the only honest grade of a six-component RAG pipeline?",
      "options": [
        "The components run on different machines",
        "The chunker is the only component that matters",
        "Production servers cannot run individual stage tests",
        "Each stage's isolated metric does not predict the system's metric because each stage interacts with the next; the only valid grade is the whole pipeline run end to end against the same qrels"
      ],
      "correct": 3,
      "explanation": "Isolated wins do not compose; the pipeline is graded on the joint output."
    },
    {
      "stage": "pre",
      "question": "What is the role of refuse-on-low-confidence in the answer generator?",
      "options": [
        "It logs every refusal to disk",
        "It returns 'I do not know' when the reranker top-1 score sits below a corpus-specific threshold, preventing hallucinated answers when the retrieved context does not address the query",
        "It blocks queries longer than 100 tokens",
        "It triggers a re-retrieval with a different chunker"
      ],
      "correct": 1,
      "explanation": "A consistently low rank-1 score is the model telling you nothing in the corpus answers this query; refuse rather than hallucinate."
    },
    {
      "stage": "check",
      "question": "Why are the rerank training triples kept disjoint from the eval queries?",
      "options": [
        "To prevent train-test leakage; if the eval queries overlap the rerank training set the eval looks magical and the regression test does not catch real drift",
        "Because RRF cannot fuse overlapping lists",
        "Because the cross-encoder requires distinct vocabularies",
        "To save GPU memory"
      ],
      "correct": 0,
      "explanation": "Strict separation is the basic hygiene; otherwise the rerank metric does not predict production behavior."
    },
    {
      "stage": "check",
      "question": "What is the purpose of the self-terminating demo's exit code?",
      "options": [
        "It is required by Python",
        "The reranker uses it as a training signal",
        "Cosmetic",
        "It encodes pass/fail against the metric thresholds; a CI smoke test can plug the demo into the pipeline and gate the merge on a non-zero exit"
      ],
      "correct": 3,
      "explanation": "Exit code 0 means thresholds met; non-zero means regression. That is the contract a CI gate consumes."
    },
    {
      "stage": "check",
      "question": "What does a citation anchor look like in this pipeline?",
      "options": [
        "JSON-RPC envelope",
        "[doc_id:chunk_index]",
        "Markdown link with the source URL",
        "The literal embedding vector"
      ],
      "correct": 1,
      "explanation": "The generator emits the (doc_id, chunk_index) tuple as an anchor so a downstream UI or auditor can map every claim back to its source chunk."
    },
    {
      "stage": "post",
      "question": "Which failure mode is invisible to the mock generator and must be tested with a real LLM before shipping?",
      "options": [
        "Latency",
        "Hallucination: the mock generator only emits text from retrieved chunks and cannot invent content; a real LLM can ignore the chunks and generate ungrounded answers",
        "Tokenization",
        "Vector index size"
      ],
      "correct": 1,
      "explanation": "The mock cannot test the generator's safety against hallucinated content; only a real model swap-in can."
    }
  ]
}