{
  "lesson": "48-distributed-fsdp-ddp",
  "title": "Distributed Data Parallel and FSDP from Scratch",
  "questions": [
    {
      "stage": "pre",
      "question": "Which two collective ops are the core of a from-scratch DDP wrapper?",
      "options": [
        "reduce_scatter only",
        "barrier and barrier",
        "send and recv",
        "broadcast at construction so every rank starts from the same parameters, and all_reduce after backward so every rank ends with the averaged gradient"
      ],
      "correct": 3,
      "explanation": "Broadcast at init keeps ranks in sync at step zero. All-reduce after backward keeps them in sync at every step that follows."
    },
    {
      "stage": "pre",
      "question": "Why does this lesson work on CPU without CUDA?",
      "options": [
        "It uses simulated tensors",
        "It does not use torch.distributed at all",
        "torch.distributed ships a gloo backend that runs collectives over plain sockets, so torch.multiprocessing workers on CPU form a real process group; only the device tag changes on a GPU box",
        "It is a stub"
      ],
      "correct": 2,
      "explanation": "Gloo is the CPU collective backend. The same call sites work on nccl on GPU; the API surface is identical."
    },
    {
      "stage": "check",
      "question": "What does the manual_all_reduce_matches_single_process test prove?",
      "options": [
        "That CUDA is unnecessary",
        "That the optimizer converges",
        "Nothing",
        "That summing per-rank gradients and dividing by world_size recovers the gradient a single process would compute on the concatenated input, within floating point noise"
      ],
      "correct": 3,
      "explanation": "The cross entropy mean reduction on the full batch equals the average of per-rank means. Sum-and-divide is the right operation; the test makes that visible."
    },
    {
      "stage": "check",
      "question": "What does the FSDP sketch in this lesson do per parameter on every forward pass?",
      "options": [
        "Computes a hash",
        "Re-initializes the parameter",
        "Nothing",
        "Pads the flat parameter to a multiple of world_size, slices, all_gathers the slices on every rank, drops the pad, and verifies the reconstruction matches the original"
      ],
      "correct": 3,
      "explanation": "Each rank owns 1/world_size of every parameter. The gather rebuilds the full tensor for compute; production FSDP overlaps the gather with the previous layer's work."
    },
    {
      "stage": "check",
      "question": "Why is the gloo all_gather sized to world_size equal-size shards even when the parameter length is not divisible by world_size?",
      "options": [
        "It speeds up the network",
        "Random choice",
        "It is a bug",
        "Gloo's all_gather requires equal output tensor sizes per rank, so the flat parameter is right-padded to a multiple of world_size before slicing and the padding is dropped after the gather"
      ],
      "correct": 3,
      "explanation": "Gloo (and nccl) all_gather expects same-size outputs. Padding is the cheapest fix; the sketch trims the result back to the original length."
    },
    {
      "stage": "post",
      "question": "Reading the demo output, param_sum_spread near zero across ranks tells you what?",
      "options": [
        "Nothing meaningful",
        "That every rank ended the run at the same parameter values, which means the broadcast at construction and the all-reduce after every backward both worked; if either was broken the spread would grow over steps",
        "That the loss is low",
        "That the network is fast"
      ],
      "correct": 1,
      "explanation": "DDP's invariant is parameter parity across ranks. The spread is the audit; the demo asserts it stays under 1e-3."
    }
  ]
}