import { isDeepStrictEqual } from "node:util"; export type Capture = { path: string; type: "string" | "number" | "boolean" }; export type RequestStep = { id: string; method: string; path: string; body?: unknown; status: number; assert: Record; capture: Record; line: number; }; export type Receipt = { id: string; line: number; status: "pass" | "fail" | "skip"; message: string; request?: { method: string; url: string; body?: unknown }; response?: { status: number; body: unknown; bytes: number }; variables: Record; }; function field(value: any, path: string): unknown { if (path === "") return value; return path .split(".") .reduce( (v, key) => v !== null && typeof v === "object" && Object.hasOwn(v, key) ? v[key] : undefined, value, ); } export function extractRequests(markdown: string): RequestStep[] { if (typeof markdown !== "string") throw Error("Tutorial must be text"); const lines = markdown.split(/\r?\n/), steps: RequestStep[] = [], ids = new Set(); for (let i = 0; i < lines.length; i++) { if (lines[i].trim() !== "```request") continue; const start = i + 1, body: string[] = []; while (++i < lines.length && lines[i].trim() !== "```") body.push(lines[i]); if (i === lines.length) throw Error("Unclosed request block at line " + start); let value; try { value = JSON.parse(body.join("\n")); } catch { throw Error("Invalid request JSON at line " + start); } const id = nonempty(value.id, "request id"); if (ids.has(id)) throw Error("Duplicate request id"); ids.add(id); const method = nonempty(value.method, "method").toUpperCase(); if (!["GET", "POST", "PUT", "PATCH", "DELETE"].includes(method)) throw Error("Unsupported HTTP method"); const path = nonempty(value.path, "path"); if (!path.startsWith("/") || path.startsWith("//") || /[\r\n\\]/.test(path)) throw Error("Request path must stay on the selected origin"); if ( !Number.isInteger(value.status) || value.status < 100 || value.status > 599 ) throw Error("Expected status must be 100..599"); const capture = value.capture ?? {}, assert = value.assert ?? {}; for (const [name, c] of Object.entries(capture) as [string, any][]) { if ( !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name) || ["constructor", "prototype", "__proto__"].includes(name) || !c || typeof c !== "object" || typeof c.path !== "string" || !["string", "number", "boolean"].includes(c.type) ) throw Error("Invalid capture declaration"); } if ( !assert || typeof assert !== "object" || Array.isArray(assert) || !capture || typeof capture !== "object" || Array.isArray(capture) ) throw Error("assert/capture must be objects"); steps.push({ id, method, path, ...(Object.hasOwn(value, "body") ? { body: value.body } : {}), status: value.status, assert, capture, line: start, }); } if (!steps.length) throw Error("No explicit request blocks"); if (steps.length > 20) throw Error("At most 20 request steps"); return steps; } export function resolveValue( value: unknown, variables: Record, ): unknown { if (typeof value === "string") { const exact = value.match(/^\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}$/); const get = (name: string) => { if (!Object.hasOwn(variables, name)) throw Error("Undeclared response variable " + name); return variables[name]; }; if (exact) return get(exact[1]); return value.replace(/\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}/g, (_, key) => String(get(key)), ); } if (Array.isArray(value)) return value.map((x) => resolveValue(x, variables)); if (value && typeof value === "object") return Object.fromEntries( Object.entries(value).map(([k, v]) => [k, resolveValue(v, variables)]), ); return value; } export function captureVariables( body: unknown, declarations: Record, previous: Record = {}, ): Record { const next = { ...previous }; for (const [name, spec] of Object.entries(declarations)) { if (Object.hasOwn(next, name)) throw Error("Capture would overwrite variable " + name); const value = field(body, spec.path); if ( typeof value !== spec.type || (typeof value === "number" && !Number.isFinite(value)) ) throw Error( "Capture " + name + " expected " + spec.type + " at " + spec.path, ); next[name] = value as string | number | boolean; } return next; } export async function runTutorial( steps: RequestStep[], baseURL: string, options: { allowNetwork?: boolean; timeoutMs?: number; maxBytes?: number; } = {}, ): Promise { const base = new URL(baseURL); if ( !["http:", "https:"].includes(base.protocol) || base.username || base.password || base.pathname !== "/" || base.search || base.hash ) throw Error("Base URL must be an HTTP(S) origin without credentials"); if (!options.allowNetwork && !["127.0.0.1", "[::1]"].includes(base.hostname)) throw Error("Network endpoint requires explicit allowNetwork"); const timeoutMs = options.timeoutMs ?? 1500, maxBytes = options.maxBytes ?? 65536; if ( !Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 30000 || !Number.isInteger(maxBytes) || maxBytes < 1 || maxBytes > 1048576 || steps.length > 20 ) throw Error("Invalid request bounds"); const receipts: Receipt[] = [], variables: Record = {}; let failed = false; for (const step of steps) { if (failed) { receipts.push({ id: step.id, line: step.line, status: "skip", message: "Earlier step failed; no request sent", variables: { ...variables }, }); continue; } const receipt: Receipt = { id: step.id, line: step.line, status: "fail", message: "", variables: { ...variables }, }; let timer: ReturnType | undefined; try { const path = step.path.replace( /\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}/g, (_, name) => { if (!Object.hasOwn(variables, name)) throw Error("Undeclared response variable " + name); return encodeURIComponent(String(variables[name])); }, ), url = new URL(path, base); if ( !path.startsWith("/") || path.startsWith("//") || url.origin !== base.origin ) throw Error("Resolved URL escaped selected origin"); const body = resolveValue(step.body, variables); if (step.method === "GET" && body !== undefined) throw Error("GET cannot carry a JSON body"); receipt.request = { method: step.method, url: url.href, ...(body === undefined ? {} : { body }), }; const controller = new AbortController(); timer = setTimeout(() => controller.abort(), timeoutMs); const response = await fetch(url, { method: step.method, headers: { accept: "application/json", ...(body === undefined ? {} : { "content-type": "application/json" }), }, ...(body === undefined ? {} : { body: JSON.stringify(body) }), redirect: "error", signal: controller.signal, }); const reader = response.body?.getReader(), chunks: Uint8Array[] = []; let bytes = 0; if (reader) while (true) { const { done, value } = await reader.read(); if (done) break; bytes += value.length; if (bytes > maxBytes) { await reader.cancel(); throw Error("Response exceeds byte limit"); } chunks.push(value); } const raw = Buffer.concat(chunks).toString("utf8"); let parsed; try { parsed = JSON.parse(raw); } catch { throw Error("Response is not JSON"); } receipt.response = { status: response.status, body: parsed, bytes }; if (response.status !== step.status) throw Error( "Expected HTTP " + step.status + ", received " + response.status, ); for (const [key, expected] of Object.entries(step.assert)) { const target = resolveValue(expected, variables); if (!isDeepStrictEqual(field(parsed, key), target)) throw Error( "Assertion failed at " + key + ": expected " + JSON.stringify(target) + ", received " + JSON.stringify(field(parsed, key)), ); } const next = captureVariables(parsed, step.capture, variables); Object.assign(variables, next); receipt.variables = { ...variables }; receipt.status = "pass"; receipt.message = "Status, assertions and captures passed"; } catch (error) { failed = true; receipt.message = (error as Error).message; } finally { if (timer) clearTimeout(timer); } receipts.push(receipt); } return receipts; } export function exportResults( steps: RequestStep[], receipts: Receipt[], ): { junit: string; html: string; corrections: any[] } { const failures = receipts.filter((x) => x.status === "fail"), skips = receipts.filter((x) => x.status === "skip"); const junit = `${receipts.map((r) => `${r.status === "fail" ? `${escapeHTML("Tutorial line " + r.line)}` : r.status === "skip" ? "" : ""}`).join("")}\n`; const corrections = failures.map((r) => ({ id: r.id, line: r.line, issue: r.message, proposal: "", approved: false, })); const html = page( "API Tutorial Runner", `

${receipts.length} steps: ${failures.length} failed, ${skips.length} skipped. Results came from real bounded HTTP requests. A failed step stops subsequent requests.

${receipts.map((r) => `

${escapeHTML(r.id)} · ${r.status}

Tutorial source line ${r.line}

${escapeHTML(r.message)}

Request and response evidence
${escapeHTML(JSON.stringify({ request: r.request, response: r.response, variables: r.variables }, null, 2))}
${r.status === "fail" ? `` : ""}
`).join("")}`, `const corrections=${embedded(corrections)};document.querySelectorAll('[data-correction]').forEach(t=>{t.oninput=()=>{corrections.find(c=>c.id===t.dataset.correction).proposal=t.value}});document.querySelectorAll('[data-approved]').forEach(c=>{c.onchange=()=>{corrections.find(x=>x.id===c.dataset.approved).approved=c.checked}});document.querySelector('#save').onclick=()=>download('corrections.json',{schemaVersion:1,corrections});`, ); return { junit, html, corrections }; } export function acceptedCorrections( receipts: Receipt[], value: unknown, ): { id: string; line: number; proposal: string }[] { const v = value as any; if (v?.schemaVersion !== 1) throw Error("Invalid correction schema"); const rows = array(v.corrections, "corrections"), ids = new Set(); return rows.flatMap((c) => { const r = receipts.find((r) => r.id === c.id && r.status === "fail"); if ( ids.has(c.id) || !r || c.line !== r.line || c.issue !== r.message || typeof c.approved !== "boolean" ) throw Error("Stale or invalid correction"); ids.add(c.id); return c.approved ? [ { id: c.id, line: c.line, proposal: nonempty(c.proposal, "approved correction"), }, ] : []; }); } export function escapeHTML(value: unknown): string { return String(value).replace( /[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[ c ]!, ); } export function embedded(value: unknown): string { return JSON.stringify(value) .replace(/${escapeHTML(title)}

AI Engineering from Scratch · local project

${escapeHTML(title)}

${content}
`; } export function array(value: unknown, label: string): any[] { if (!Array.isArray(value)) throw new Error(label + " must be an array"); return value; } export function nonempty(value: unknown, label: string): string { if (typeof value !== "string" || !value.trim()) throw new Error(label + " must be a nonempty string"); return value; }