// Gateway concurrency benchmark tests cover CLI parsing and bounded percentile summaries. import { spawnSync } from "node:child_process"; import { createServer as createHttpServer } from "node:http"; import { createServer as createRawServer, type Socket } from "node:net"; import { performance } from "node:perf_hooks"; import { describe, expect, it } from "vitest"; import { testing } from "../../scripts/bench-gateway-concurrency.ts"; describe("gateway concurrency benchmark script", () => { it("parses benchmark controls without booting a gateway", () => { expect( testing.parseOptions([ "--concurrency", "12", "--runs", "2", "--warmup", "0", "--cadence-ms", "50", "--timeout-ms", "90000", "--output", "concurrency.json", "--json", ]), ).toMatchObject({ cadenceMs: 50, concurrency: 12, json: true, output: "concurrency.json", runs: 2, timeoutMs: 90_000, warmup: 0, }); expect(() => testing.parseOptions(["--concurrency", "65"])).toThrow( "--concurrency must be at most 64", ); expect(() => testing.parseOptions(["--runs", "2", "--runs", "3"])).toThrow( "--runs was provided more than once", ); expect(() => testing.parseOptions(["--wat"])).toThrow("Unknown argument: --wat"); }); it("reports p50, p95, p99, and max with nearest-rank percentiles", () => { expect(testing.summarizeNumbers([100, 1, 4, 2, 3])).toEqual({ count: 5, max: 100, p50: 3, p95: 100, p99: 100, }); expect(testing.summarizeNumbers([])).toBeNull(); }); it("bounds an accepted turn wait by the benchmark deadline", async () => { const calls: Array<{ method: string; params: unknown; timeoutMs?: number }> = []; const rpc = async (method: string, params: unknown, timeoutMs?: number): Promise => { calls.push({ method, params, timeoutMs }); return ( method === "agent" ? { runId: "run-1", status: "accepted" } : { status: "timeout" } ) as T; }; await expect(testing.runTurn(rpc, 0, performance.now() + 2_000)).rejects.toThrow( "agent 1 did not complete", ); const wait = calls.find((call) => call.method === "agent.wait"); expect(wait?.params).toMatchObject({ runId: "run-1" }); const serverTimeoutMs = (wait?.params as { timeoutMs?: unknown } | undefined)?.timeoutMs; expect(serverTimeoutMs).toBe(0); expect(wait?.timeoutMs).toEqual(expect.any(Number)); expect(Number.isInteger(wait?.timeoutMs)).toBe(true); expect(wait?.timeoutMs).toBeGreaterThan(serverTimeoutMs as number); expect(wait?.timeoutMs).toBeLessThanOrEqual(2_000); }); it("gives every gateway sample a fresh pre-warmup timeout budget", async () => { const deadlines: number[] = []; const sample = { controlUi: [], durationMs: 10, probeWarmup: { durationMs: 2, samples: [] }, readyz: [], sessionsList: [], turnCount: 8, turnsDurationMs: 5, }; const runs = await testing.runBenchmarkSamples({ now: (() => { const values = [1_000, 9_000]; return () => values.shift() ?? 9_000; })(), options: testing.parseOptions(["--runs", "1", "--warmup", "1", "--timeout-ms", "5000"]), runSample: async ({ deadlineAt }) => { deadlines.push(deadlineAt); return sample; }, }); expect(deadlines).toEqual([6_000, 14_000]); expect(runs).toEqual([sample]); }); it("preserves HTTP and RPC failures in baseline probe diagnostics", async () => { const probeOrder: string[] = []; const server = createHttpServer((req, res) => { probeOrder.push(req.url ?? "missing-url"); res.statusCode = req.url === "/readyz" ? 503 : 200; res.end(req.url === "/readyz" ? '{"status":"starting"}' : "not html"); }); await new Promise((resolve) => { server.listen(0, "127.0.0.1", resolve); }); const address = server.address(); if (!address || typeof address === "string") { server.close(); throw new Error("expected HTTP test server address"); } try { const sample = await testing.sampleGateway({ deadlineAt: performance.now() + 5_000, port: address.port, rpc: async () => { probeOrder.push("sessions.list"); throw new Error("sessions.list failed: unauthorized"); }, runStartedAt: performance.now(), serial: true, }); expect(probeOrder).toEqual(["/readyz", "/", "sessions.list"]); expect(sample.readyz).toMatchObject({ error: null, ok: false, status: 503 }); expect(sample.controlUi).toMatchObject({ error: "response body did not contain "gateway output", readStderrTail: () => testing.tailLines("old\nfirst retained\nlast retained\n", 2), }, { readOutput: () => "mock output" }, ); expect(failure).toMatch( /readyz: ok=false status=503 latencyMs=\d+\.\d error=none\n {2}sessionsList: ok=false status=n\/a latencyMs=\d+\.\d error="sessions\.list failed: unauthorized"\n {2}controlUi: ok=false status=200 latencyMs=\d+\.\d error="response body did not contain samples.shift() ?? healthyFast, targetMs: 100, }); expect(warmed.samples).toHaveLength(3); } finally { server.close(); } }); it("bounds trickled response bodies by the benchmark deadline", async () => { const sockets = new Set(); let bodyChunksSent = 0; let serverEndedResponse = false; const server = createRawServer((socket) => { sockets.add(socket); socket.setNoDelay(true); socket.on("error", () => {}); socket.once("close", () => sockets.delete(socket)); socket.once("data", () => { socket.write( "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n ", ); bodyChunksSent += 1; const interval = setInterval(() => { socket.write(" "); bodyChunksSent += 1; }, 10); const endTimer = setTimeout(() => { serverEndedResponse = true; socket.end(); }, 500); socket.once("close", () => { clearInterval(interval); clearTimeout(endTimer); }); }); }); await new Promise((resolve) => { server.listen(0, "127.0.0.1", () => resolve()); }); const address = server.address(); if (!address || typeof address === "string") { server.close(); throw new Error("expected raw HTTP test server address"); } const startedAt = performance.now(); try { await expect( testing.requestHttp({ accept: "application/json", deadlineAt: startedAt + 150, path: "/readyz", port: address.port, }), ).rejects.toThrow("/readyz request timed out"); expect(bodyChunksSent).toBeGreaterThan(1); expect(serverEndedResponse).toBe(false); } finally { for (const socket of sockets) { socket.destroy(); } await new Promise((resolve) => { server.close(() => resolve()); }); } }); it("reuses one connection for sequential successful HTTP samples", async () => { let connectionCount = 0; const server = createHttpServer((request, response) => { response.setHeader( "content-type", request.url === "/readyz" ? "application/json" : "text/html", ); response.end(request.url === "/readyz" ? '{"status":"ok"}' : ""); }); server.on("connection", () => { connectionCount += 1; }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); try { const address = server.address(); if (!address || typeof address === "string") { throw new Error("expected HTTP test server address"); } const deadlineAt = performance.now() + 5_000; await testing.requestHttp({ accept: "application/json", deadlineAt, path: "/readyz", port: address.port, }); await testing.requestHttp({ accept: "text/html", deadlineAt, path: "/", port: address.port, }); expect(connectionCount).toBe(1); } finally { server.closeAllConnections(); await new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); }); } }); it("loads through native Node TypeScript stripping", () => { const result = spawnSync(process.execPath, ["scripts/bench-gateway-concurrency.ts", "--help"], { cwd: process.cwd(), encoding: "utf8", }); expect(result.status).toBe(0); expect(result.stdout).toContain("OpenClaw Gateway concurrency benchmark"); }); it("ends CLI failures with the required wrapper marker", () => { const result = spawnSync(process.execPath, ["scripts/bench-gateway-concurrency.ts", "--wat"], { cwd: process.cwd(), encoding: "utf8", }); expect(result.status).toBe(1); expect(result.stderr.trim().split("\n").at(-1)).toBe( "[bench-gateway-concurrency] FAILED (exit 1)", ); }); });