{
  "id": "sglang-101",
  "title": "SGLang 101",
  "subtitle": "Serving language models with SGLang, from one request to a fleet, with the judgment the flag list leaves out",
  "summary": "How SGLang turns GPU memory and time into tokens, why each feature exists, and which defaults to trust. Local listings come from recorded runs of SGLang 0.5.22.",
  "audience": "You are an engineer who has called a model API and wants to serve an open model yourself. You know what a token and a GPU are. You have not read a serving engine's scheduler or cache, and you want to know which settings matter before you pay for them.",
  "edition": "2026.10",
  "status": "ready",
  "pin": {
    "subject": "SGLang",
    "version": "0.5.22",
    "source": "https://github.com/sgl-project/sglang",
    "commit": "baa9befa381511cd4d56f83980951e5a45fffc6d",
    "date": "2026-10-13",
    "verified": "2026-10-16"
  },
  "outcomes": [
    "Install SGLang, serve a model, and get the first answer from both of its APIs in about ten minutes.",
    "Size the KV pool for your GPU, and fix a server that runs out of memory.",
    "Tune the server for your traffic: shared prompts, long documents, many users, or FP8.",
    "Benchmark your own server, and watch it in production with its metrics.",
    "Choose how to scale to many GPUs, and judge a published benchmark before you trust it."
  ],
  "palette": [
    {
      "hue": "blue",
      "meaning": "the SGLang server process: tokenizer manager, scheduler, detokenizer"
    },
    {
      "hue": "violet",
      "meaning": "the model and its forward pass: kernels, attention backends, CUDA graphs"
    },
    {
      "hue": "green",
      "meaning": "the KV cache: pool slots, radix tree nodes, cached tokens"
    },
    {
      "hue": "plum",
      "meaning": "model compute: prefill and decode steps"
    },
    {
      "hue": "olive",
      "meaning": "the client and the network: HTTP, the OpenAI API, the benchmark client"
    },
    {
      "hue": "indigo",
      "meaning": "streams and messages: ZMQ messages and streamed tokens"
    },
    {
      "hue": "amber",
      "meaning": "lifecycle: request states and scheduler phases"
    },
    {
      "hue": "teal",
      "meaning": "durable state: host memory and storage tiers, weights on disk"
    },
    {
      "hue": "rose",
      "meaning": "failure: out of memory, retraction, aborts, crashes"
    },
    {
      "hue": "grey",
      "meaning": "outside SGLang: the operating system, other programs, the GPU driver"
    }
  ],
  "sources": [
    {
      "name": "SGLang source at the release tag",
      "path": "v0.5.22 (baa9befa)",
      "usedFor": "every default and mechanism; files listed in each Sources line"
    },
    {
      "name": "Captured runs",
      "path": "capture/out/ (SGLang 0.5.22, MLX, Qwen3-0.6B-4bit, macOS 26.2 arm64)",
      "usedFor": "every listing and every number marked as measured"
    },
    {
      "name": "SGLang GitHub issues and pull requests",
      "path": "github.com/sgl-project/sglang",
      "usedFor": "incidents and the design history of each feature"
    },
    {
      "name": "Papers",
      "path": "SGLang (arXiv 2312.07104) and the papers the code implements",
      "usedFor": "the original design of RadixAttention and the methods SGLang implements"
    },
    {
      "name": "SGLang documentation at the release tag",
      "path": "docs/ at baa9befa",
      "usedFor": "documented behaviour, checked against the code"
    },
    {
      "name": "Independent field reports",
      "path": "engineering blogs, benchmarks, and audited results by people outside the SGLang team",
      "usedFor": "GPU behaviour this manual could not run, labeled with the author and date"
    },
    {
      "name": "SGLang team and vendor reports",
      "path": "LMSYS, RadixArk, and hardware vendor posts",
      "usedFor": "claims, always labeled as the author's claim"
    }
  ],
  "quoteSources": {
    "code": null,
    "docs": null,
    "paper": null,
    "issue": null,
    "field": null,
    "team": null
  },
  "capture": {
    "run": [
      "python3",
      "capture/run.py"
    ],
    "check": [
      "python3",
      "capture/run.py",
      "--check"
    ],
    "timeoutSeconds": 1800
  },
  "front": "front.md",
  "plate": {
    "figure": "plate-sglang-request",
    "title": "One radix tree on one L4",
    "caption": "On our L4, the system prompt is one shared node of the radix tree, and each repeat request attaches below it. The repeats had 2,998 of 2,999 tokens cached, and the first token came after 82.5 ms instead of 2,143.9 ms. From capture/gpu/out-l4/cache-experiment.txt and capture/gpu/out-l4/server-default.log."
  },
  "parts": [
    {
      "number": 1,
      "title": "Start",
      "thesis": "Install SGLang, serve a model, check that the server is healthy, and learn the two phases and four numbers behind every request.",
      "summary": "Install with uv or Docker, the first answer from both APIs, the health checks and startup lines, then prefill against decode and the four numbers.",
      "accent": "blue",
      "sections": [
        {
          "id": "1.1",
          "file": "sections/1-1-install-sglang-and-serve-your-first-model.md"
        },
        {
          "id": "1.2",
          "file": "sections/1-2-check-that-the-server-is-healthy.md"
        },
        {
          "id": "1.3",
          "file": "sections/1-3-what-the-server-does-with-your-request.md"
        }
      ]
    },
    {
      "number": 2,
      "title": "The Request Path",
      "thesis": "A request crosses three processes, and the scheduler between them decides who runs, who waits, and who gets thrown out.",
      "summary": "sglang serve, the scheduler loop and its policies, admission and retraction, the overlap scheduler, and the response path.",
      "accent": "amber",
      "sections": [
        {
          "id": "2.1",
          "file": "sections/2-1-sglang-serve-and-the-three-processes.md"
        },
        {
          "id": "2.2",
          "file": "sections/2-2-the-scheduler-loop-and-schedule-policy.md"
        },
        {
          "id": "2.3",
          "file": "sections/2-3-admission-and-retraction.md"
        },
        {
          "id": "2.4",
          "file": "sections/2-4-the-overlap-scheduler.md"
        },
        {
          "id": "2.5",
          "file": "sections/2-5-streaming-stop-strings-and-the-openai-api.md"
        }
      ]
    },
    {
      "number": 3,
      "title": "The KV Cache",
      "thesis": "Memory, not compute, decides how many users one GPU can serve, and the radix tree decides how much of that memory you pay for twice.",
      "summary": "Bytes per token, startup sizing, RadixCache, prefix-friendly prompts, eviction and HiCache, and hybrid models.",
      "accent": "green",
      "sections": [
        {
          "id": "3.1",
          "file": "sections/3-1-kv-bytes-per-token-and-the-kv-pool.md"
        },
        {
          "id": "3.2",
          "file": "sections/3-2-mem-fraction-static-and-startup-sizing.md"
        },
        {
          "id": "3.3",
          "file": "sections/3-3-radixcache.md"
        },
        {
          "id": "3.4",
          "file": "sections/3-4-prefix-friendly-prompts.md"
        },
        {
          "id": "3.5",
          "file": "sections/3-5-eviction-policies-and-hicache.md"
        },
        {
          "id": "3.6",
          "file": "sections/3-6-hybrid-models-and-the-cache.md"
        }
      ]
    },
    {
      "number": 4,
      "title": "Each Forward Pass",
      "thesis": "Most decode time is launch overhead and memory traffic, so SGLang picks graphs, kernels, and draft tokens for your GPU before you ask.",
      "summary": "ForwardMode and CUDA graphs, attention backends, quantization, speculative decoding, structured outputs, and deterministic inference.",
      "accent": "violet",
      "sections": [
        {
          "id": "4.1",
          "file": "sections/4-1-forwardmode-and-cuda-graphs.md"
        },
        {
          "id": "4.2",
          "file": "sections/4-2-attention-backend.md"
        },
        {
          "id": "4.3",
          "file": "sections/4-3-quantization.md"
        },
        {
          "id": "4.4",
          "file": "sections/4-4-speculative-decoding.md"
        },
        {
          "id": "4.5",
          "file": "sections/4-5-structured-outputs-with-xgrammar.md"
        },
        {
          "id": "4.6",
          "file": "sections/4-6-deterministic-inference-and-logprobs.md"
        }
      ]
    },
    {
      "number": 5,
      "title": "Measuring a Server",
      "thesis": "A throughput number without a latency target describes nothing, so measure the knee before you tune anything.",
      "summary": "sglang.benchmark.serving, the /metrics endpoint, and how to read benchmarks that other people publish.",
      "accent": "olive",
      "sections": [
        {
          "id": "5.1",
          "file": "sections/5-1-sglang-benchmark-serving.md"
        },
        {
          "id": "5.2",
          "file": "sections/5-2-the-metrics-endpoint.md"
        },
        {
          "id": "5.3",
          "file": "sections/5-3-reading-other-peoples-benchmarks.md"
        }
      ]
    },
    {
      "number": 6,
      "title": "One Server Becomes a Fleet",
      "thesis": "Split a model across GPUs only after one GPU is saturated, and split prefill from decode only after you measured that prefill hurts.",
      "summary": "Tensor, data, attention-data, and expert parallelism, MoE serving, prefill-decode disaggregation, and the model gateway.",
      "accent": "indigo",
      "sections": [
        {
          "id": "6.1",
          "file": "sections/6-1-tp-size-dp-size-and-attn-dp-size.md"
        },
        {
          "id": "6.2",
          "file": "sections/6-2-moe-serving-deepep-and-eplb.md"
        },
        {
          "id": "6.3",
          "file": "sections/6-3-prefill-decode-disaggregation.md"
        },
        {
          "id": "6.4",
          "file": "sections/6-4-sgl-model-gateway.md"
        }
      ]
    },
    {
      "number": 7,
      "title": "Production and Training",
      "thesis": "Operate SGLang with its defaults written down, and expect reinforcement learning to treat the server as part of the training loop.",
      "summary": "Deploying and tuning in order, and SGLang as the rollout engine of Miles.",
      "accent": "plum",
      "sections": [
        {
          "id": "7.1",
          "file": "sections/7-1-operating-sglang.md"
        },
        {
          "id": "7.2",
          "file": "sections/7-2-sglang-as-an-rl-rollout-engine.md"
        }
      ]
    }
  ],
  "reference": {
    "title": "Reference",
    "thesis": "The flags, metrics, incidents, terms, and sources this manual uses, in one place.",
    "accent": "grey",
    "sections": [
      {
        "id": "R.1",
        "file": "sections/r-1-flags-and-defaults.md"
      },
      {
        "id": "R.2",
        "file": "sections/r-2-metrics.md"
      },
      {
        "id": "R.3",
        "file": "sections/r-3-incident-index.md"
      },
      {
        "id": "R.4",
        "file": "sections/r-4-glossary.md"
      },
      {
        "id": "R.5",
        "file": "sections/r-5-sources.md"
      },
      {
        "id": "R.6",
        "file": "sections/r-6-figure-index.md"
      },
      {
        "id": "R.7",
        "file": "sections/r-7-how-this-manual-was-made.md"
      }
    ]
  },
  "related": [
    "phases/17-infrastructure-and-production/04-vllm-serving-internals",
    "phases/17-infrastructure-and-production/08-inference-metrics-goodput",
    "phases/17-infrastructure-and-production/05-eagle3-speculative-decoding",
    "phases/17-infrastructure-and-production/09-production-quantization",
    "phases/17-infrastructure-and-production/28-self-hosted-serving-selection",
    "phases/10-llms-from-scratch/12-inference-optimization",
    "phases/11-llm-engineering/15-prompt-caching"
  ]
}