feat(gateway): track event-loop degradation persistence and add concurrency benchmark (#118193)

* feat(gateway): persist event loop degradation metrics

* fix(gateway): bound concurrency benchmark turn waits

test/scripts/bench-gateway-concurrency.test.ts

@@ -0,0 +1,89 @@
+// Gateway concurrency benchmark tests cover CLI parsing and bounded percentile summaries.
+import { spawnSync } from "node:child_process";
+import { performance } from "node:perf_hooks";
+import { describe, expect, it } from "vitest";
+import { testing } from "../../scripts/bench-gateway-concurrency.ts";
+
+describe("gateway concurrency benchmark script", () => {
+  it("parses benchmark controls without booting a gateway", () => {
+    expect(
+      testing.parseOptions([
+        "--concurrency",
+        "12",
+        "--runs",
+        "2",
+        "--warmup",
+        "0",
+        "--cadence-ms",
+        "50",
+        "--timeout-ms",
+        "90000",
+        "--output",
+        "concurrency.json",
+        "--json",
+      ]),
+    ).toMatchObject({
+      cadenceMs: 50,
+      concurrency: 12,
+      json: true,
+      output: "concurrency.json",
+      runs: 2,
+      timeoutMs: 90_000,
+      warmup: 0,
+    });
+    expect(() => testing.parseOptions(["--concurrency", "65"])).toThrow(
+      "--concurrency must be at most 64",
+    );
+    expect(() => testing.parseOptions(["--runs", "2", "--runs", "3"])).toThrow(
+      "--runs was provided more than once",
+    );
+    expect(() => testing.parseOptions(["--wat"])).toThrow("Unknown argument: --wat");
+  });
+
+  it("reports p50, p95, p99, and max with nearest-rank percentiles", () => {
+    expect(testing.summarizeNumbers([100, 1, 4, 2, 3])).toEqual({
+      count: 5,
+      max: 100,
+      p50: 3,
+      p95: 100,
+      p99: 100,
+    });
+    expect(testing.summarizeNumbers([])).toBeNull();
+  });
+
+  it("bounds an accepted turn wait by the benchmark deadline", async () => {
+    const calls: Array<{ method: string; params: unknown; timeoutMs?: number }> = [];
+    const rpc = async <T>(method: string, params: unknown, timeoutMs?: number): Promise<T> => {
+      calls.push({ method, params, timeoutMs });
+      return (
+        method === "agent" ? { runId: "run-1", status: "accepted" } : { status: "timeout" }
+      ) as T;
+    };
+
+    await expect(testing.runTurn(rpc, 0, performance.now() + 2_000)).rejects.toThrow(
+      "agent 1 did not complete",
+    );
+
+    const wait = calls.find((call) => call.method === "agent.wait");
+    expect(wait?.params).toMatchObject({ runId: "run-1" });
+    const serverTimeoutMs = (wait?.params as { timeoutMs?: unknown }).timeoutMs;
+    expect(serverTimeoutMs).toBe(0);
+    expect(wait?.timeoutMs).toEqual(expect.any(Number));
+    expect(Number.isInteger(wait?.timeoutMs)).toBe(true);
+    expect(wait?.timeoutMs).toBeGreaterThan(serverTimeoutMs as number);
+    expect(wait?.timeoutMs).toBeLessThanOrEqual(2_000);
+  });
+
+  it("ends CLI failures with the required wrapper marker", () => {
+    const result = spawnSync(
+      process.execPath,
+      ["--import", "tsx", "scripts/bench-gateway-concurrency.ts", "--wat"],
+      { cwd: process.cwd(), encoding: "utf8" },
+    );
+
+    expect(result.status).toBe(1);
+    expect(result.stderr.trim().split("\n").at(-1)).toBe(
+      "[bench-gateway-concurrency] FAILED (exit 1)",
+    );
+  });
+});

test/scripts/check-gateway-cpu-scenarios.test.ts

@@ -50,6 +50,21 @@ function writeQaSuiteSummary(
   );
 }
 
+function writeConcurrencyReport(outputDir: string): void {
+  writeFileSync(
+    path.join(outputDir, "gateway-concurrency-bench.json"),
+    `${JSON.stringify({
+      mode: "mock-streaming-agent",
+      runs: [{ turnCount: 8 }],
+      summary: {
+        eventLoopDelayP99Ms: { max: 20 },
+        sessionsListLatencyMs: { p99: 30 },
+        controlUiLatencyMs: { p99: 25 },
+      },
+    })}\n`,
+  );
+}
+
 afterEach(() => {
   for (const root of tempRoots.splice(0)) {
     rmSync(root, { recursive: true, force: true });
@@ -160,6 +175,9 @@ describe("gateway CPU scenario guard", () => {
         if (args.includes("scripts/bench-gateway-startup.ts")) {
           writeFileSync(startupOutput, `${JSON.stringify({ results: [{ id: "default" }] })}\n`);
         }
+        if (args.includes("scripts/bench-gateway-concurrency.ts")) {
+          writeConcurrencyReport(outputDir);
+        }
         return { status: 0 };
       },
     });
@@ -168,10 +186,13 @@ describe("gateway CPU scenario guard", () => {
     expect(calls.map((call) => call.args[0])).toEqual([
       "scripts/ensure-cli-startup-build.mjs",
       "--import",
+      "--import",
     ]);
     expect(calls[1]?.args).toContain("scripts/bench-gateway-startup.ts");
+    expect(calls[2]?.args).toContain("scripts/bench-gateway-concurrency.ts");
     expect(calls[0]?.env?.PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN).toBe("false");
     expect(calls[1]?.env?.PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN).toBe("false");
+    expect(calls[2]?.env?.PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN).toBe("false");
   });
 
   it("fails successful startup benches that do not write a report", async () => {
@@ -248,6 +269,7 @@ describe("gateway CPU scenario guard", () => {
     expect(result.summary.steps).toEqual([
       { name: "startup build", signal: null, status: 1 },
       { name: "startup bench", signal: null, status: 1 },
+      { name: "concurrency bench", signal: null, status: 1 },
     ]);
   });
 
@@ -273,6 +295,7 @@ describe("gateway CPU scenario guard", () => {
     expect(result.summary.steps).toEqual([
       { name: "startup build", error: "spawn ENOENT", signal: null, status: 1 },
       { name: "startup bench", signal: null, status: 1 },
+      { name: "concurrency bench", signal: null, status: 1 },
     ]);
   });