mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
feat(gateway): track event-loop degradation persistence and add concurrency benchmark (#118193)
* feat(gateway): persist event loop degradation metrics * fix(gateway): bound concurrency benchmark turn waits
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
c79dcff608
commit
3f3ceb2def
@@ -0,0 +1,719 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
// Bench Gateway Concurrency script measures gateway probes during synthetic streaming turns.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { request } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { PROTOCOL_VERSION } from "../packages/gateway-protocol/src/index.js";
|
||||
import { applyMockOpenAiModelConfig } from "./e2e/lib/fixtures/mock-openai-config.mjs";
|
||||
import { delay, stopChild } from "./lib/gateway-bench-child.ts";
|
||||
import { getFreePort } from "./lib/gateway-bench-probes.ts";
|
||||
import {
|
||||
BASE_GATEWAY_BENCH_CONFIG,
|
||||
buildGatewayBenchChildArgs,
|
||||
CliArgumentError,
|
||||
createGatewayBenchEnv,
|
||||
hasFlag,
|
||||
hasHelpFlag,
|
||||
parseFlagValue,
|
||||
parseNonNegativeInt,
|
||||
parsePositiveInt,
|
||||
resolveEntry,
|
||||
resolveOutputPath,
|
||||
validateCliArgs,
|
||||
waitForInitialProbe,
|
||||
writeGatewayBenchConfig,
|
||||
} from "./lib/gateway-bench-runtime.ts";
|
||||
import { createGatewayWsClient } from "./lib/gateway-ws-client.ts";
|
||||
|
||||
type MetricSummary = {
|
||||
count: number;
|
||||
max: number;
|
||||
p50: number;
|
||||
p95: number;
|
||||
p99: number;
|
||||
};
|
||||
|
||||
type TimedProbe = {
|
||||
atMs: number;
|
||||
latencyMs: number;
|
||||
ok: boolean;
|
||||
};
|
||||
|
||||
type ReadyProbe = TimedProbe & {
|
||||
cpuCoreRatio: number | null;
|
||||
degraded: boolean | null;
|
||||
degradedSinceMs: number | null;
|
||||
delayP99Ms: number | null;
|
||||
status: number;
|
||||
utilization: number | null;
|
||||
};
|
||||
|
||||
type GatewayRpc = <T>(method: string, params: unknown, timeoutMs?: number) => Promise<T>;
|
||||
|
||||
type BenchmarkRun = {
|
||||
controlUi: TimedProbe[];
|
||||
durationMs: number;
|
||||
readyz: ReadyProbe[];
|
||||
sessionsList: TimedProbe[];
|
||||
turnCount: number;
|
||||
turnsDurationMs: number;
|
||||
};
|
||||
|
||||
type CliOptions = {
|
||||
cadenceMs: number;
|
||||
concurrency: number;
|
||||
entry: string;
|
||||
json: boolean;
|
||||
output?: string;
|
||||
runs: number;
|
||||
timeoutMs: number;
|
||||
warmup: number;
|
||||
};
|
||||
|
||||
const DEFAULT_CADENCE_MS = 100;
|
||||
const DEFAULT_CONCURRENCY = 8;
|
||||
const DEFAULT_ENTRY = "dist/entry.js";
|
||||
const DEFAULT_RUNS = 1;
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const DEFAULT_WARMUP = 0;
|
||||
const MOCK_RESPONSE_CHUNK_DELAY_MS = 1_000;
|
||||
const MAX_CONCURRENCY = 64;
|
||||
const MAX_RUNS = 20;
|
||||
const MAX_WARMUP = 10;
|
||||
const MAX_SAMPLES_PER_RUN = 2_048;
|
||||
const MAX_HTTP_BODY_BYTES = 1_048_576;
|
||||
const HTTP_TIMEOUT_MS = 20_000;
|
||||
const AGENT_WAIT_RPC_GRACE_MS = 5_000;
|
||||
const BOOLEAN_FLAGS = new Set(["--help", "-h", "--json"]);
|
||||
const VALUE_FLAGS = new Set([
|
||||
"--cadence-ms",
|
||||
"--concurrency",
|
||||
"--entry",
|
||||
"--output",
|
||||
"--runs",
|
||||
"--timeout-ms",
|
||||
"--warmup",
|
||||
]);
|
||||
|
||||
function parseBoundedPositiveInt(
|
||||
raw: string | undefined,
|
||||
fallback: number,
|
||||
label: string,
|
||||
max: number,
|
||||
): number {
|
||||
const value = parsePositiveInt(raw, fallback, label);
|
||||
if (value > max) {
|
||||
throw new CliArgumentError(`${label} must be at most ${max}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseBoundedNonNegativeInt(
|
||||
raw: string | undefined,
|
||||
fallback: number,
|
||||
label: string,
|
||||
max: number,
|
||||
): number {
|
||||
const value = parseNonNegativeInt(raw, fallback, label);
|
||||
if (value > max) {
|
||||
throw new CliArgumentError(`${label} must be at most ${max}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseOptions(argv: string[] = process.argv.slice(2)): CliOptions {
|
||||
validateCliArgs(argv, { booleanFlags: BOOLEAN_FLAGS, valueFlags: VALUE_FLAGS });
|
||||
return {
|
||||
cadenceMs: parseBoundedPositiveInt(
|
||||
parseFlagValue(argv, "--cadence-ms"),
|
||||
DEFAULT_CADENCE_MS,
|
||||
"--cadence-ms",
|
||||
5_000,
|
||||
),
|
||||
concurrency: parseBoundedPositiveInt(
|
||||
parseFlagValue(argv, "--concurrency"),
|
||||
DEFAULT_CONCURRENCY,
|
||||
"--concurrency",
|
||||
MAX_CONCURRENCY,
|
||||
),
|
||||
entry: resolveEntry(parseFlagValue(argv, "--entry"), DEFAULT_ENTRY),
|
||||
json: hasFlag(argv, "--json"),
|
||||
output: resolveOutputPath(parseFlagValue(argv, "--output")),
|
||||
runs: parseBoundedPositiveInt(parseFlagValue(argv, "--runs"), DEFAULT_RUNS, "--runs", MAX_RUNS),
|
||||
timeoutMs: parseBoundedPositiveInt(
|
||||
parseFlagValue(argv, "--timeout-ms"),
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
"--timeout-ms",
|
||||
10 * 60_000,
|
||||
),
|
||||
warmup: parseBoundedNonNegativeInt(
|
||||
parseFlagValue(argv, "--warmup"),
|
||||
DEFAULT_WARMUP,
|
||||
"--warmup",
|
||||
MAX_WARMUP,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function printUsage(): void {
|
||||
console.log(`OpenClaw Gateway concurrency benchmark
|
||||
|
||||
Usage:
|
||||
pnpm test:gateway:concurrency -- [options]
|
||||
node --import tsx scripts/bench-gateway-concurrency.ts [options]
|
||||
|
||||
Options:
|
||||
--concurrency <n> Concurrent synthetic streaming turns (default: ${DEFAULT_CONCURRENCY})
|
||||
--runs <n> Measured gateway runs (default: ${DEFAULT_RUNS})
|
||||
--warmup <n> Warmup gateway runs (default: ${DEFAULT_WARMUP})
|
||||
--cadence-ms <ms> Probe cadence (default: ${DEFAULT_CADENCE_MS})
|
||||
--timeout-ms <ms> Whole benchmark wall-clock cap (default: ${DEFAULT_TIMEOUT_MS})
|
||||
--entry <path> Gateway CLI entry file (default: ${DEFAULT_ENTRY})
|
||||
--output <path> Write machine-readable JSON to a file
|
||||
--json Emit machine-readable JSON
|
||||
--help, -h Show this text
|
||||
`);
|
||||
}
|
||||
|
||||
function percentile(sorted: readonly number[], percentileValue: number): number {
|
||||
const index = Math.max(
|
||||
0,
|
||||
Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1),
|
||||
);
|
||||
return sorted[index] ?? 0;
|
||||
}
|
||||
|
||||
function summarizeNumbers(values: readonly number[]): MetricSummary | null {
|
||||
const sorted = values.filter(Number.isFinite).toSorted((a, b) => a - b);
|
||||
if (sorted.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
count: sorted.length,
|
||||
max: sorted.at(-1) ?? 0,
|
||||
p50: percentile(sorted, 50),
|
||||
p95: percentile(sorted, 95),
|
||||
p99: percentile(sorted, 99),
|
||||
};
|
||||
}
|
||||
|
||||
function remainingMs(deadlineAt: number): number {
|
||||
return Math.max(0, deadlineAt - performance.now());
|
||||
}
|
||||
|
||||
function requireRemainingMs(deadlineAt: number, label: string): number {
|
||||
const remaining = remainingMs(deadlineAt);
|
||||
if (remaining <= 0) {
|
||||
throw new Error(`benchmark timed out while ${label}`);
|
||||
}
|
||||
return remaining;
|
||||
}
|
||||
|
||||
async function requestHttp(params: {
|
||||
accept: string;
|
||||
deadlineAt: number;
|
||||
path: string;
|
||||
port: number;
|
||||
}): Promise<{ body: string; latencyMs: number; status: number }> {
|
||||
const startedAt = performance.now();
|
||||
const timeoutMs = Math.max(
|
||||
1,
|
||||
Math.min(HTTP_TIMEOUT_MS, requireRemainingMs(params.deadlineAt, `requesting ${params.path}`)),
|
||||
);
|
||||
return await new Promise((resolve, reject) => {
|
||||
const req = request(
|
||||
{
|
||||
headers: { accept: params.accept },
|
||||
host: "127.0.0.1",
|
||||
method: "GET",
|
||||
path: params.path,
|
||||
port: params.port,
|
||||
timeout: timeoutMs,
|
||||
},
|
||||
(res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let bytes = 0;
|
||||
res.on("data", (chunk: Buffer) => {
|
||||
bytes += chunk.length;
|
||||
if (bytes > MAX_HTTP_BODY_BYTES) {
|
||||
req.destroy(new Error(`${params.path} response exceeded ${MAX_HTTP_BODY_BYTES} bytes`));
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
res.on("end", () => {
|
||||
resolve({
|
||||
body: Buffer.concat(chunks).toString("utf8"),
|
||||
latencyMs: performance.now() - startedAt,
|
||||
status: res.statusCode ?? 0,
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
req.once("error", reject);
|
||||
req.once("timeout", () => req.destroy(new Error(`${params.path} request timed out`)));
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function numberOrNull(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function captureChildOutput(child: ChildProcessWithoutNullStreams): () => string {
|
||||
let output = "";
|
||||
const append = (chunk: Buffer) => {
|
||||
output = `${output}${chunk.toString("utf8")}`.slice(-64 * 1_024);
|
||||
};
|
||||
child.stdout.on("data", append);
|
||||
child.stderr.on("data", append);
|
||||
return () => output;
|
||||
}
|
||||
|
||||
async function waitForMockServer(port: number, deadlineAt: number): Promise<void> {
|
||||
let lastError: unknown;
|
||||
while (remainingMs(deadlineAt) > 0) {
|
||||
try {
|
||||
const result = await requestHttp({
|
||||
accept: "application/json",
|
||||
deadlineAt,
|
||||
path: "/health",
|
||||
port,
|
||||
});
|
||||
if (result.status === 200) {
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
await delay(Math.min(25, remainingMs(deadlineAt)));
|
||||
}
|
||||
const detail =
|
||||
lastError instanceof Error
|
||||
? lastError.message
|
||||
: typeof lastError === "string"
|
||||
? lastError
|
||||
: "timeout";
|
||||
throw new Error(`mock provider did not become healthy: ${detail}`);
|
||||
}
|
||||
|
||||
async function waitForGatewayDispatchReady(
|
||||
readOutput: () => string,
|
||||
deadlineAt: number,
|
||||
): Promise<void> {
|
||||
while (remainingMs(deadlineAt) > 0) {
|
||||
if (readOutput().includes("startup trace: sidecars.ready ")) {
|
||||
return;
|
||||
}
|
||||
await delay(Math.min(25, remainingMs(deadlineAt)));
|
||||
}
|
||||
throw new Error("gateway did not finish dispatch-ready sidecars");
|
||||
}
|
||||
|
||||
function buildConfig(root: string, mockPort: number, concurrency: number): string {
|
||||
const controlUiRoot = path.join(root, "control-ui");
|
||||
mkdirSync(controlUiRoot, { recursive: true });
|
||||
copyFileSync(
|
||||
path.join(process.cwd(), "ui", "index.html"),
|
||||
path.join(controlUiRoot, "index.html"),
|
||||
);
|
||||
|
||||
const config = structuredClone(BASE_GATEWAY_BENCH_CONFIG) as Record<string, unknown>;
|
||||
config.gateway = {
|
||||
...(config.gateway as Record<string, unknown>),
|
||||
controlUi: { enabled: true, root: controlUiRoot },
|
||||
};
|
||||
applyMockOpenAiModelConfig(config, { mockPort, modelRef: "openai/gpt-5.6-luna" });
|
||||
const agents = config.agents as Record<string, unknown>;
|
||||
agents.defaults = {
|
||||
...(agents.defaults as Record<string, unknown>),
|
||||
maxConcurrent: concurrency,
|
||||
};
|
||||
return writeGatewayBenchConfig(root, config, {});
|
||||
}
|
||||
|
||||
async function connectGateway(port: number, deadlineAt: number) {
|
||||
const client = createGatewayWsClient({
|
||||
handshakeTimeoutMs: Math.min(8_000, requireRemainingMs(deadlineAt, "connecting WebSocket")),
|
||||
openTimeoutMs: Math.min(8_000, requireRemainingMs(deadlineAt, "opening WebSocket")),
|
||||
url: `ws://127.0.0.1:${port}`,
|
||||
});
|
||||
await client.waitOpen();
|
||||
|
||||
const requestRpc = async <T>(
|
||||
method: string,
|
||||
params: unknown,
|
||||
requestedTimeoutMs?: number,
|
||||
): Promise<T> => {
|
||||
const response = await client.request(
|
||||
method,
|
||||
params,
|
||||
Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
requestedTimeoutMs ?? 65_000,
|
||||
requireRemainingMs(deadlineAt, `waiting for ${method}`),
|
||||
),
|
||||
),
|
||||
);
|
||||
if (!response.ok) {
|
||||
const message =
|
||||
response.error && typeof response.error === "object" && "message" in response.error
|
||||
? String(response.error.message)
|
||||
: JSON.stringify(response.error);
|
||||
throw new Error(`${method} failed: ${message}`);
|
||||
}
|
||||
return response.payload as T;
|
||||
};
|
||||
|
||||
await requestRpc("connect", {
|
||||
minProtocol: PROTOCOL_VERSION,
|
||||
maxProtocol: PROTOCOL_VERSION,
|
||||
client: {
|
||||
id: "gateway-client",
|
||||
displayName: "gateway-concurrency-benchmark",
|
||||
version: "1.0.0",
|
||||
platform: process.platform,
|
||||
mode: "backend",
|
||||
},
|
||||
role: "operator",
|
||||
scopes: ["operator.read", "operator.write", "operator.admin"],
|
||||
caps: [],
|
||||
});
|
||||
return { close: client.close, request: requestRpc };
|
||||
}
|
||||
|
||||
async function runTurn(rpc: GatewayRpc, index: number, deadlineAt: number): Promise<void> {
|
||||
const requestedRunId = randomUUID();
|
||||
const started = await rpc<{ runId?: string; status?: string }>("agent", {
|
||||
sessionKey: `agent:main:gateway-concurrency-${index + 1}`,
|
||||
message: `Reply with benchmark stream ${index + 1}.`,
|
||||
deliver: false,
|
||||
idempotencyKey: requestedRunId,
|
||||
});
|
||||
if (started.status === "ok") {
|
||||
return;
|
||||
}
|
||||
if (started.status !== "accepted") {
|
||||
throw new Error(`agent ${index + 1} was not accepted: ${JSON.stringify(started)}`);
|
||||
}
|
||||
const remaining = requireRemainingMs(deadlineAt, `waiting for agent ${index + 1} completion`);
|
||||
const waitTimeoutMs = Math.max(
|
||||
0,
|
||||
Math.min(60_000, Math.floor(remaining - AGENT_WAIT_RPC_GRACE_MS)),
|
||||
);
|
||||
const rpcTimeoutMs = Math.min(65_000, Math.max(1, Math.ceil(remaining)));
|
||||
const completed = await rpc<{ status?: string }>(
|
||||
"agent.wait",
|
||||
{
|
||||
runId: started.runId ?? requestedRunId,
|
||||
timeoutMs: waitTimeoutMs,
|
||||
},
|
||||
rpcTimeoutMs,
|
||||
);
|
||||
if (completed.status !== "ok") {
|
||||
throw new Error(`agent ${index + 1} did not complete: ${JSON.stringify(completed)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function sampleGateway(params: {
|
||||
deadlineAt: number;
|
||||
port: number;
|
||||
rpc: GatewayRpc;
|
||||
runStartedAt: number;
|
||||
}): Promise<{ controlUi: TimedProbe; readyz: ReadyProbe; sessionsList: TimedProbe }> {
|
||||
const atMs = performance.now() - params.runStartedAt;
|
||||
const safeHttpProbe = async (pathValue: string, accept: string) => {
|
||||
const startedAt = performance.now();
|
||||
try {
|
||||
return {
|
||||
...(await requestHttp({
|
||||
accept,
|
||||
deadlineAt: params.deadlineAt,
|
||||
path: pathValue,
|
||||
port: params.port,
|
||||
})),
|
||||
ok: true,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
body: "",
|
||||
latencyMs: performance.now() - startedAt,
|
||||
ok: false,
|
||||
status: 0,
|
||||
};
|
||||
}
|
||||
};
|
||||
const [readyz, controlUi, sessions] = await Promise.all([
|
||||
safeHttpProbe("/readyz", "application/json"),
|
||||
safeHttpProbe("/", "text/html"),
|
||||
(async () => {
|
||||
const startedAt = performance.now();
|
||||
try {
|
||||
const payload = await params.rpc("sessions.list", {}, HTTP_TIMEOUT_MS);
|
||||
return { latencyMs: performance.now() - startedAt, ok: true, payload };
|
||||
} catch {
|
||||
return { latencyMs: performance.now() - startedAt, ok: false, payload: null };
|
||||
}
|
||||
})(),
|
||||
]);
|
||||
const readyBody = (() => {
|
||||
if (readyz.status !== 200) {
|
||||
return {};
|
||||
}
|
||||
try {
|
||||
return JSON.parse(readyz.body) as { eventLoop?: Record<string, unknown> };
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
})();
|
||||
const eventLoop = readyBody.eventLoop;
|
||||
return {
|
||||
controlUi: {
|
||||
atMs,
|
||||
latencyMs: controlUi.latencyMs,
|
||||
ok: controlUi.ok && controlUi.status === 200 && controlUi.body.includes("<html"),
|
||||
},
|
||||
readyz: {
|
||||
atMs,
|
||||
latencyMs: readyz.latencyMs,
|
||||
ok: readyz.ok && readyz.status === 200,
|
||||
status: readyz.status,
|
||||
degraded: typeof eventLoop?.degraded === "boolean" ? eventLoop.degraded : null,
|
||||
degradedSinceMs: numberOrNull(eventLoop?.degradedSinceMs),
|
||||
delayP99Ms: numberOrNull(eventLoop?.delayP99Ms),
|
||||
utilization: numberOrNull(eventLoop?.utilization),
|
||||
cpuCoreRatio: numberOrNull(eventLoop?.cpuCoreRatio),
|
||||
},
|
||||
sessionsList: { atMs, latencyMs: sessions.latencyMs, ok: sessions.ok },
|
||||
};
|
||||
}
|
||||
|
||||
async function runGatewaySample(options: {
|
||||
cadenceMs: number;
|
||||
concurrency: number;
|
||||
deadlineAt: number;
|
||||
entry: string;
|
||||
}): Promise<BenchmarkRun> {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-gateway-concurrency-"));
|
||||
const [port, mockPort] = await Promise.all([getFreePort(), getFreePort()]);
|
||||
const runStartedAt = performance.now();
|
||||
let gateway: ChildProcessWithoutNullStreams | undefined;
|
||||
let mockProvider: ChildProcessWithoutNullStreams | undefined;
|
||||
let client: Awaited<ReturnType<typeof connectGateway>> | undefined;
|
||||
let gatewayOutput = () => "";
|
||||
let mockOutput = () => "";
|
||||
|
||||
try {
|
||||
const configPath = buildConfig(root, mockPort, options.concurrency);
|
||||
mockProvider = spawn(process.execPath, ["scripts/e2e/mock-openai-server.mjs"], {
|
||||
cwd: process.cwd(),
|
||||
detached: process.platform !== "win32",
|
||||
env: {
|
||||
LANG: process.env.LANG ?? "en_US.UTF-8",
|
||||
PATH: process.env.PATH,
|
||||
MOCK_PORT: String(mockPort),
|
||||
MOCK_RESPONSE_CHUNK_DELAY_MS: String(MOCK_RESPONSE_CHUNK_DELAY_MS),
|
||||
SUCCESS_MARKER: "OpenClaw gateway concurrency benchmark streaming response.",
|
||||
},
|
||||
});
|
||||
mockOutput = captureChildOutput(mockProvider);
|
||||
await waitForMockServer(mockPort, options.deadlineAt);
|
||||
|
||||
gateway = spawn(process.execPath, buildGatewayBenchChildArgs(options.entry, port), {
|
||||
cwd: process.cwd(),
|
||||
detached: process.platform !== "win32",
|
||||
env: {
|
||||
...createGatewayBenchEnv(root, configPath, {
|
||||
caseEnv: { OPENCLAW_SKIP_CHANNELS: "1" },
|
||||
}),
|
||||
OPENAI_API_KEY: "gateway-concurrency-benchmark",
|
||||
},
|
||||
});
|
||||
gatewayOutput = captureChildOutput(gateway);
|
||||
const ready = await waitForInitialProbe({
|
||||
deadlineAt: options.deadlineAt,
|
||||
isDone: () => gateway?.exitCode != null || gateway?.signalCode != null,
|
||||
path: "/readyz",
|
||||
port,
|
||||
startAt: runStartedAt,
|
||||
});
|
||||
if (ready.status !== 200) {
|
||||
throw new Error(`gateway did not become ready\n${gatewayOutput()}`);
|
||||
}
|
||||
await waitForGatewayDispatchReady(gatewayOutput, options.deadlineAt);
|
||||
client = await connectGateway(port, options.deadlineAt);
|
||||
const rpc = client.request;
|
||||
const baseline = await sampleGateway({
|
||||
deadlineAt: options.deadlineAt,
|
||||
port,
|
||||
rpc,
|
||||
runStartedAt,
|
||||
});
|
||||
if (!baseline.readyz.ok || !baseline.sessionsList.ok || !baseline.controlUi.ok) {
|
||||
throw new Error("gateway probes did not pass before concurrent load");
|
||||
}
|
||||
|
||||
const controlUi: TimedProbe[] = [];
|
||||
const readyz: ReadyProbe[] = [];
|
||||
const sessionsList: TimedProbe[] = [];
|
||||
let turnsDone = false;
|
||||
const turnsStartedAt = performance.now();
|
||||
const turns = Promise.all(
|
||||
Array.from({ length: options.concurrency }, (_, index) =>
|
||||
runTurn(rpc, index, options.deadlineAt),
|
||||
),
|
||||
).finally(() => {
|
||||
turnsDone = true;
|
||||
});
|
||||
const sampler = (async () => {
|
||||
for (;;) {
|
||||
const sampleStartedAt = performance.now();
|
||||
const sample = await sampleGateway({
|
||||
deadlineAt: options.deadlineAt,
|
||||
port,
|
||||
rpc,
|
||||
runStartedAt,
|
||||
});
|
||||
readyz.push(sample.readyz);
|
||||
sessionsList.push(sample.sessionsList);
|
||||
controlUi.push(sample.controlUi);
|
||||
if (turnsDone || readyz.length >= MAX_SAMPLES_PER_RUN) {
|
||||
break;
|
||||
}
|
||||
await delay(
|
||||
Math.min(
|
||||
Math.max(0, options.cadenceMs - (performance.now() - sampleStartedAt)),
|
||||
requireRemainingMs(options.deadlineAt, "sampling gateway load"),
|
||||
),
|
||||
);
|
||||
}
|
||||
})();
|
||||
await Promise.all([turns, sampler]);
|
||||
const turnsDurationMs = performance.now() - turnsStartedAt;
|
||||
|
||||
return {
|
||||
controlUi,
|
||||
durationMs: performance.now() - runStartedAt,
|
||||
readyz,
|
||||
sessionsList,
|
||||
turnCount: options.concurrency,
|
||||
turnsDurationMs,
|
||||
};
|
||||
} catch (error) {
|
||||
const detail = [
|
||||
error instanceof Error ? error.message : String(error),
|
||||
gatewayOutput() ? `gateway output:\n${gatewayOutput()}` : "",
|
||||
mockOutput() ? `mock provider output:\n${mockOutput()}` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
throw new Error(detail, { cause: error });
|
||||
} finally {
|
||||
client?.close();
|
||||
if (gateway) {
|
||||
await stopChild(gateway);
|
||||
}
|
||||
if (mockProvider) {
|
||||
await stopChild(mockProvider);
|
||||
}
|
||||
rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 });
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeRuns(runs: readonly BenchmarkRun[]) {
|
||||
const readyz = runs.flatMap((run) => run.readyz);
|
||||
return {
|
||||
controlUiFailedSamples: runs.flatMap((run) => run.controlUi).filter((sample) => !sample.ok)
|
||||
.length,
|
||||
controlUiLatencyMs: summarizeNumbers(
|
||||
runs.flatMap((run) => run.controlUi.map((sample) => sample.latencyMs)),
|
||||
),
|
||||
cpuCoreRatio: summarizeNumbers(
|
||||
readyz.flatMap((sample) => (sample.cpuCoreRatio == null ? [] : [sample.cpuCoreRatio])),
|
||||
),
|
||||
degradedSamples: readyz.filter((sample) => sample.degraded === true).length,
|
||||
eventLoopDelayP99Ms: summarizeNumbers(
|
||||
readyz.flatMap((sample) => (sample.delayP99Ms == null ? [] : [sample.delayP99Ms])),
|
||||
),
|
||||
eventLoopUtilization: summarizeNumbers(
|
||||
readyz.flatMap((sample) => (sample.utilization == null ? [] : [sample.utilization])),
|
||||
),
|
||||
readyzLatencyMs: summarizeNumbers(readyz.map((sample) => sample.latencyMs)),
|
||||
readyzFailedSamples: readyz.filter((sample) => !sample.ok).length,
|
||||
sampleCount: readyz.length,
|
||||
sessionsListLatencyMs: summarizeNumbers(
|
||||
runs.flatMap((run) => run.sessionsList.map((sample) => sample.latencyMs)),
|
||||
),
|
||||
sessionsListFailedSamples: runs
|
||||
.flatMap((run) => run.sessionsList)
|
||||
.filter((sample) => !sample.ok).length,
|
||||
turnsDurationMs: summarizeNumbers(runs.map((run) => run.turnsDurationMs)),
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const argv = process.argv.slice(2);
|
||||
if (hasHelpFlag(argv)) {
|
||||
printUsage();
|
||||
return;
|
||||
}
|
||||
const options = parseOptions(argv);
|
||||
const deadlineAt = performance.now() + options.timeoutMs;
|
||||
const runs: BenchmarkRun[] = [];
|
||||
const total = options.runs + options.warmup;
|
||||
for (let index = 0; index < total; index += 1) {
|
||||
requireRemainingMs(deadlineAt, "starting gateway run");
|
||||
const run = await runGatewaySample({ ...options, deadlineAt });
|
||||
if (index >= options.warmup) {
|
||||
runs.push(run);
|
||||
console.error(
|
||||
`[bench-gateway-concurrency] run ${runs.length}/${options.runs}: turns=${run.turnCount} samples=${run.readyz.length} duration=${run.durationMs.toFixed(1)}ms`,
|
||||
);
|
||||
} else {
|
||||
console.error(
|
||||
`[bench-gateway-concurrency] warmup ${index + 1}/${options.warmup}: duration=${run.durationMs.toFixed(1)}ms`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const payload = {
|
||||
cadenceMs: options.cadenceMs,
|
||||
concurrency: options.concurrency,
|
||||
entry: options.entry,
|
||||
generatedAt: new Date().toISOString(),
|
||||
mode: "mock-streaming-agent",
|
||||
runs,
|
||||
summary: summarizeRuns(runs),
|
||||
};
|
||||
if (options.output) {
|
||||
mkdirSync(path.dirname(options.output), { recursive: true });
|
||||
writeFileSync(options.output, `${JSON.stringify(payload, null, 2)}\n`);
|
||||
}
|
||||
if (options.json || !options.output) {
|
||||
console.log(JSON.stringify(payload, null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
parseOptions,
|
||||
runTurn,
|
||||
summarizeNumbers,
|
||||
summarizeRuns,
|
||||
};
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
void main()
|
||||
.catch((error: unknown) => {
|
||||
console.error(error instanceof CliArgumentError ? error.message : (error as Error)?.stack);
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(() => {
|
||||
if (process.exitCode && process.exitCode !== 0) {
|
||||
console.error(`[bench-gateway-concurrency] FAILED (exit ${process.exitCode})`);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -33,6 +33,12 @@ const SINGLE_VALUE_FLAGS = new Set([
|
||||
]);
|
||||
const DEFAULT_CPU_CORE_WARN = 0.9;
|
||||
const DEFAULT_HOT_WALL_WARN_MS = 30_000;
|
||||
const DEFAULT_GATEWAY_CONCURRENCY = 8;
|
||||
// Local N=8 mock-stream p99s were 1.3-1.5s on the shared maintainer host;
|
||||
// 3s leaves roughly 2x headroom while still flagging a material regression.
|
||||
const CONCURRENCY_EVENT_LOOP_DELAY_P99_WARN_MS = 3_000;
|
||||
const CONCURRENCY_RPC_P99_WARN_MS = 3_000;
|
||||
const CONCURRENCY_CONTROL_UI_P99_WARN_MS = 3_000;
|
||||
const PRIVATE_QA_REQUIRED_DIST_ENTRIES = [
|
||||
"dist/plugin-sdk/qa-lab.js",
|
||||
"dist/plugin-sdk/qa-runtime.js",
|
||||
@@ -191,6 +197,75 @@ function readStartupReport(startupOutput) {
|
||||
}
|
||||
}
|
||||
|
||||
function validateConcurrencyReport(report) {
|
||||
if (!report || typeof report !== "object" || Array.isArray(report)) {
|
||||
return "concurrency report must be a JSON object";
|
||||
}
|
||||
if (report.mode !== "mock-streaming-agent") {
|
||||
return "concurrency report has an unexpected mode";
|
||||
}
|
||||
if (!Array.isArray(report.runs) || report.runs.length === 0) {
|
||||
return "concurrency report has no measured runs";
|
||||
}
|
||||
if (!report.summary || typeof report.summary !== "object") {
|
||||
return "concurrency report missing summary";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function readConcurrencyReport(concurrencyOutput) {
|
||||
if (!fs.existsSync(concurrencyOutput)) {
|
||||
return {
|
||||
diagnosticFailure: "concurrency-report-missing",
|
||||
diagnosticDetail: `expected concurrency bench report at ${concurrencyOutput}`,
|
||||
report: null,
|
||||
};
|
||||
}
|
||||
try {
|
||||
const report = readJsonIfExists(concurrencyOutput);
|
||||
const invalidReason = validateConcurrencyReport(report);
|
||||
return invalidReason
|
||||
? {
|
||||
diagnosticFailure: "concurrency-report-invalid",
|
||||
diagnosticDetail: invalidReason,
|
||||
report: null,
|
||||
}
|
||||
: { diagnosticFailure: null, diagnosticDetail: null, report };
|
||||
} catch (error) {
|
||||
return {
|
||||
diagnosticFailure: "concurrency-report-invalid",
|
||||
diagnosticDetail: error instanceof Error ? error.message : String(error),
|
||||
report: null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function collectConcurrencyWarnings(report) {
|
||||
if (!report?.summary) {
|
||||
return [];
|
||||
}
|
||||
const candidates = [
|
||||
{
|
||||
kind: "event-loop-delay-p99",
|
||||
value: report.summary.eventLoopDelayP99Ms?.max,
|
||||
threshold: CONCURRENCY_EVENT_LOOP_DELAY_P99_WARN_MS,
|
||||
},
|
||||
{
|
||||
kind: "sessions-list-p99",
|
||||
value: report.summary.sessionsListLatencyMs?.p99,
|
||||
threshold: CONCURRENCY_RPC_P99_WARN_MS,
|
||||
},
|
||||
{
|
||||
kind: "control-ui-p99",
|
||||
value: report.summary.controlUiLatencyMs?.p99,
|
||||
threshold: CONCURRENCY_CONTROL_UI_P99_WARN_MS,
|
||||
},
|
||||
];
|
||||
return candidates.flatMap((candidate) =>
|
||||
typeof candidate.value === "number" && candidate.value > candidate.threshold ? [candidate] : [],
|
||||
);
|
||||
}
|
||||
|
||||
function runStep(name, command, args, options = {}, params = {}) {
|
||||
console.error(`[gateway-cpu] start ${name}`);
|
||||
const spawn = params.spawnSync ?? defaultSpawnSync;
|
||||
@@ -287,6 +362,7 @@ async function runGatewayCpuScenarios(options, params = {}) {
|
||||
fs.mkdirSync(options.outputDir, { recursive: true });
|
||||
|
||||
const startupOutput = path.join(options.outputDir, "gateway-startup-bench.json");
|
||||
const concurrencyOutput = path.join(options.outputDir, "gateway-concurrency-bench.json");
|
||||
const qaOutputDir = path.join(options.outputDir, "qa-suite");
|
||||
const qaSummaryPath = path.join(qaOutputDir, "qa-suite-summary.json");
|
||||
const qaState = options.skipQa ? null : createQaState(options.outputDir);
|
||||
@@ -329,6 +405,29 @@ async function runGatewayCpuScenarios(options, params = {}) {
|
||||
)
|
||||
: { name: "startup bench", signal: null, status: 1 },
|
||||
);
|
||||
steps.push(
|
||||
startupBuild.status === 0
|
||||
? runStep(
|
||||
"concurrency bench",
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"scripts/bench-gateway-concurrency.ts",
|
||||
"--concurrency",
|
||||
String(DEFAULT_GATEWAY_CONCURRENCY),
|
||||
"--runs",
|
||||
String(options.runs),
|
||||
"--warmup",
|
||||
String(options.warmup),
|
||||
"--output",
|
||||
concurrencyOutput,
|
||||
],
|
||||
{ env: baseEnv },
|
||||
params,
|
||||
)
|
||||
: { name: "concurrency bench", signal: null, status: 1 },
|
||||
);
|
||||
}
|
||||
|
||||
let privateQaBuildFailed = false;
|
||||
@@ -373,6 +472,15 @@ async function runGatewayCpuScenarios(options, params = {}) {
|
||||
? (startupReportResult?.diagnosticFailure ?? null)
|
||||
: null;
|
||||
const startup = startupReportResult?.report ?? null;
|
||||
const concurrencyReportResult = options.skipStartup
|
||||
? null
|
||||
: readConcurrencyReport(concurrencyOutput);
|
||||
const concurrencyReportFailure =
|
||||
steps.find((step) => step.name === "concurrency bench")?.status === 0
|
||||
? (concurrencyReportResult?.diagnosticFailure ?? null)
|
||||
: null;
|
||||
const concurrency = concurrencyReportResult?.report ?? null;
|
||||
const concurrencyWarnings = collectConcurrencyWarnings(concurrency);
|
||||
const qaSummaryResult = options.skipQa ? null : readQaSuiteSummary(qaSummaryPath);
|
||||
const qaSummaryFailure =
|
||||
qaStep?.status === 0 ? (qaSummaryResult?.diagnosticFailure ?? null) : null;
|
||||
@@ -387,6 +495,7 @@ async function runGatewayCpuScenarios(options, params = {}) {
|
||||
generatedAt: new Date().toISOString(),
|
||||
outputDir: options.outputDir,
|
||||
startupOutput: fs.existsSync(startupOutput) ? startupOutput : null,
|
||||
concurrencyOutput: fs.existsSync(concurrencyOutput) ? concurrencyOutput : null,
|
||||
qaSummary: fs.existsSync(qaSummaryPath) ? qaSummaryPath : null,
|
||||
...(startupReportFailure
|
||||
? {
|
||||
@@ -400,6 +509,12 @@ async function runGatewayCpuScenarios(options, params = {}) {
|
||||
qaSummaryFailureDetail: qaSummaryResult?.diagnosticDetail ?? null,
|
||||
}
|
||||
: {}),
|
||||
...(concurrencyReportFailure
|
||||
? {
|
||||
concurrencyReportFailure,
|
||||
concurrencyReportFailureDetail: concurrencyReportResult?.diagnosticDetail ?? null,
|
||||
}
|
||||
: {}),
|
||||
options: {
|
||||
startupCases: options.startupCases,
|
||||
qaScenarios: options.qaScenarios,
|
||||
@@ -407,10 +522,17 @@ async function runGatewayCpuScenarios(options, params = {}) {
|
||||
warmup: options.warmup,
|
||||
cpuCoreWarn: options.cpuCoreWarn,
|
||||
hotWallWarnMs: options.hotWallWarnMs,
|
||||
concurrency: DEFAULT_GATEWAY_CONCURRENCY,
|
||||
concurrencyWarnThresholds: {
|
||||
eventLoopDelayP99Ms: CONCURRENCY_EVENT_LOOP_DELAY_P99_WARN_MS,
|
||||
sessionsListP99Ms: CONCURRENCY_RPC_P99_WARN_MS,
|
||||
controlUiP99Ms: CONCURRENCY_CONTROL_UI_P99_WARN_MS,
|
||||
},
|
||||
qaStateDir: qaState?.stateDir ?? null,
|
||||
},
|
||||
steps,
|
||||
observations,
|
||||
concurrencyWarnings,
|
||||
};
|
||||
const summaryPath = path.join(options.outputDir, "summary.json");
|
||||
fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);
|
||||
@@ -430,12 +552,23 @@ async function runGatewayCpuScenarios(options, params = {}) {
|
||||
if (startupReportFailure) {
|
||||
console.error(`[gateway-cpu] fail startup report: ${startupReportResult?.diagnosticDetail}`);
|
||||
}
|
||||
for (const warning of concurrencyWarnings) {
|
||||
console.error(
|
||||
`[gateway-cpu] warn ${warning.kind}: ${warning.value}ms > ${warning.threshold}ms`,
|
||||
);
|
||||
}
|
||||
if (concurrencyReportFailure) {
|
||||
console.error(
|
||||
`[gateway-cpu] fail concurrency report: ${concurrencyReportResult?.diagnosticDetail}`,
|
||||
);
|
||||
}
|
||||
|
||||
const exitCode =
|
||||
steps.some((step) => step.status !== 0) ||
|
||||
observations.length > 0 ||
|
||||
qaSummaryFailure ||
|
||||
startupReportFailure
|
||||
startupReportFailure ||
|
||||
concurrencyReportFailure
|
||||
? 1
|
||||
: 0;
|
||||
return { exitCode, summary };
|
||||
|
||||
Reference in New Issue
Block a user