{
  "lesson": "43-hdf5-tokenized-corpus",
  "title": "HDF5 Tokenized Corpus",
  "questions": [
    {
      "stage": "pre",
      "question": "Why store the tokenized corpus in a resizable, chunked HDF5 dataset rather than JSONL?",
      "options": [
        "HDF5 has a smaller logo",
        "JSONL is deprecated",
        "Random reasons",
        "HDF5 offers chunked, integer-only, cache-friendly reads with O(1) random access; JSONL forces the parser onto every hot-path read"
      ],
      "correct": 3,
      "explanation": "The dataloader wants a flat token stream with addressable slices; HDF5 supplies that natively."
    },
    {
      "stage": "check",
      "question": "What is the right rule for sizing the HDF5 chunk vs the trainer's window_size?",
      "options": [
        "Pick the largest chunk that fits in RAM",
        "Always pick chunk=1",
        "Set chunk to a multiple of window_size so each sample lands inside one or two chunks and reads stay page-cache aligned",
        "Pick chunk equal to the document length"
      ],
      "correct": 2,
      "explanation": "Mismatched chunks halve throughput because every sample straddles two chunks instead of one."
    },
    {
      "stage": "check",
      "question": "Why is token_count stored as an HDF5 attribute and not implied from dataset shape?",
      "options": [
        "The trailing chunk may be partially full; without the explicit attribute the reader walks past the real end into zero-padded tokens and the model learns to predict zero",
        "Shapes are slow to read",
        "Attributes are mandatory",
        "Attributes are pretty"
      ],
      "correct": 0,
      "explanation": "The chunked layout sometimes leaves trailing slots; the attribute carries the truth."
    },
    {
      "stage": "check",
      "question": "What does the writer accomplish by buffering tokens to chunk_size before each resize?",
      "options": [
        "Writes are contiguous and chunk-aligned, avoiding the fragmentation that follows one-token-at-a-time resizes",
        "Reduces vocabulary size",
        "Speeds up sha256",
        "Saves a few CPU cycles"
      ],
      "correct": 0,
      "explanation": "Buffer-then-extend at chunk size is the only way to keep HDF5 chunks contiguous on disk."
    },
    {
      "stage": "post",
      "question": "Why is swmr=True opened on the reader side, not on the writer side?",
      "options": [
        "Security",
        "Random preference",
        "Performance",
        "Single-Writer-Multiple-Reader mode lets dataloader workers share the file safely; the writer needs no special mode because it writes one shard per process"
      ],
      "correct": 3,
      "explanation": "Mixing the two modes is the most common cause of 'file is locked' failures."
    },
    {
      "stage": "post",
      "question": "What does the boundary token id exist to make explicit?",
      "options": [
        "Compression",
        "Vocabulary size limits",
        "Random data",
        "Document boundaries within a packed window so the model is not silently trained on cross-document concatenation"
      ],
      "correct": 3,
      "explanation": "Without an explicit boundary the trainer learns boundaries as noise; with it the boundary becomes a usable separator."
    }
  ]
}