openclaw/scripts/bench-gateway-startup.ts

// Bench Gateway Startup script supports OpenClaw repository automation.
import { spawn } from "node:child_process";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { performance } from "node:perf_hooks";
import { pathToFileURL } from "node:url";
import { delay, stopChild } from "./lib/gateway-bench-child.ts";
import {
  getFreePort,
  parseProcessRssKb,
  readProcessRssMb,
  readProcessTreeCpuMs,
} from "./lib/gateway-bench-probes.ts";
import {
  BASE_GATEWAY_BENCH_CONFIG,
  buildGatewayBenchChildArgs,
  classifyGatewayReadyLog,
  CliArgumentError,
  collectOutputLines,
  collectTraceLine,
  createGatewayBenchEnv,
  flushOutputLineBuffers,
  formatMb,
  formatMs,
  formatStats,
  hasFlag,
  hasHelpFlag,
  parseFlagValue,
  parseNonNegativeInt,
  parsePositiveInt,
  parseRepeatableFlag,
  resolveCases as resolveGatewayBenchCases,
  resolveEntry as resolveGatewayBenchEntry,
  resolveOutputPath,
  STALLED_CATALOG_MODEL_ID,
  STALLED_CATALOG_PROVIDER_ID,
  summarizeNumbers,
  type InitialProbeResult,
  type SummaryStats,
  validateCliArgs as validateGatewayBenchCliArgs,
  writeGatewayBenchConfig,
  writePluginFixtures,
  waitForInitialProbe as waitForProbe,
} from "./lib/gateway-bench-runtime.ts";
import { selectSlowStartupTraceDurations } from "./lib/gateway-startup-trace-ranking.js";

type GatewayBenchCase = {
  agentTopology?: "single" | "shared-eleven-plus-distinct-one";
  completionTracePhase?: string;
  config: Record<string, unknown>;
  env?: Record<string, string>;
  id: string;
  name: string;
  pluginActivationOnStartup?: boolean;
  pluginCount?: number;
  providerCatalogStallMs?: number;
  providerStaticCatalogModelCount?: number;
  providerStaticCatalogStallMs?: number;
};

type ProbeResult = InitialProbeResult;

type GatewaySample = {
  completionMs: number | null;
  cpuCoreRatio: number | null;
  cpuMs: number | null;
  exitedBeforeTeardown?: boolean;
  exitCode: number | null;
  firstOutputMs: number | null;
  gatewayReadyLogLine: string | null;
  gatewayReadyLogMs: number | null;
  healthz: ProbeResult;
  httpListenLogLine: string | null;
  httpListenLogMs: number | null;
  maxRssMb: number | null;
  outputTail: string;
  readyz: ProbeResult;
  signal: string | null;
  startupTrace: Record<string, number>;
};

type CaseResult = {
  id: string;
  name: string;
  samples: GatewaySample[];
  summary: {
    completionMs: SummaryStats | null;
    firstOutputMs: SummaryStats | null;
    cpuCoreRatio: SummaryStats | null;
    cpuMs: SummaryStats | null;
    gatewayReadyLogMs: SummaryStats | null;
    healthzMs: SummaryStats | null;
    httpListenLogMs: SummaryStats | null;
    maxRssMb: SummaryStats | null;
    readyzMs: SummaryStats | null;
    startupTrace: Record<string, SummaryStats>;
  };
};

type BenchmarkFailure = {
  id: string;
  reason: string;
  sampleIndex: number;
};

type CliOptions = {
  cases: GatewayBenchCase[];
  cpuProfDir?: string;
  entry: string;
  heapProfDir?: string;
  json: boolean;
  output?: string;
  runs: number;
  timeoutMs: number;
  warmup: number;
};

const DEFAULT_RUNS = 5;
const DEFAULT_WARMUP = 1;
const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_ENTRY = "dist/entry.js";
const BOOLEAN_FLAGS = new Set(["--help", "-h", "--json"]);
const VALUE_FLAGS = new Set([
  "--case",
  "--cpu-prof-dir",
  "--entry",
  "--heap-prof-dir",
  "--output",
  "--runs",
  "--timeout-ms",
  "--warmup",
]);

const BASE_CONFIG = BASE_GATEWAY_BENCH_CONFIG;

const GATEWAY_CASES: readonly GatewayBenchCase[] = [
  {
    id: "default",
    name: "gateway default",
    config: BASE_CONFIG,
  },
  {
    id: "skipChannels",
    name: "gateway, skip channels",
    env: { OPENCLAW_SKIP_CHANNELS: "1" },
    config: BASE_CONFIG,
  },
  {
    id: "preparedRuntimeCatalogStall",
    name: "gateway, prepared runtime with CPU-stalling live catalog",
    env: { OPENCLAW_SKIP_CHANNELS: "1" },
    providerCatalogStallMs: 2_000,
    config: {
      ...BASE_CONFIG,
      agents: {
        defaults: {
          model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` },
          models: {
            [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: {
              agentRuntime: { id: "openclaw" },
            },
          },
        },
      },
    },
  },
  {
    id: "preparedRuntimeScaleOne",
    name: "gateway, prepared runtime scale with one agent",
    agentTopology: "single",
    completionTracePhase: "sidecars.ready",
    env: { OPENCLAW_SKIP_CHANNELS: "1" },
    providerStaticCatalogModelCount: 64,
    providerStaticCatalogStallMs: 100,
    config: {
      ...BASE_CONFIG,
      agents: {
        defaults: {
          model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` },
          models: {
            [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: {
              agentRuntime: { id: "openclaw" },
            },
          },
        },
      },
    },
  },
  {
    id: "preparedRuntimeScaleMany",
    name: "gateway, prepared runtime scale with 11 shared-workspace agents and one distinct",
    agentTopology: "shared-eleven-plus-distinct-one",
    completionTracePhase: "sidecars.ready",
    env: { OPENCLAW_SKIP_CHANNELS: "1" },
    providerStaticCatalogModelCount: 64,
    providerStaticCatalogStallMs: 100,
    config: {
      ...BASE_CONFIG,
      agents: {
        defaults: {
          model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` },
          models: {
            [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: {
              agentRuntime: { id: "openclaw" },
            },
          },
        },
      },
    },
  },
  {
    id: "oneInternalHook",
    name: "gateway, one configured internal hook",
    env: { OPENCLAW_SKIP_CHANNELS: "1" },
    config: {
      ...BASE_CONFIG,
      hooks: {
        internal: {
          entries: {
            "session-memory": { enabled: true },
          },
        },
      },
    },
  },
  {
    id: "allInternalHooks",
    name: "gateway, all internal hooks",
    env: { OPENCLAW_SKIP_CHANNELS: "1" },
    config: {
      ...BASE_CONFIG,
      hooks: {
        internal: {
          enabled: true,
        },
      },
    },
  },
  {
    id: "fiftyPlugins",
    name: "gateway, 50 manifest plugins",
    env: { OPENCLAW_SKIP_CHANNELS: "1" },
    pluginActivationOnStartup: true,
    pluginCount: 50,
    config: BASE_CONFIG,
  },
  {
    id: "fiftyStartupLazyPlugins",
    name: "gateway, 50 startup-lazy manifest plugins",
    env: { OPENCLAW_SKIP_CHANNELS: "1" },
    pluginActivationOnStartup: false,
    pluginCount: 50,
    config: BASE_CONFIG,
  },
] as const;

function validateCliArgs(argv: string[]): void {
  validateGatewayBenchCliArgs(argv, {
    booleanFlags: BOOLEAN_FLAGS,
    repeatableValueFlags: new Set(["--case"]),
    valueFlags: VALUE_FLAGS,
  });
}

function resolveEntry(raw: string | undefined): string {
  return resolveGatewayBenchEntry(raw, DEFAULT_ENTRY);
}

function resolveCases(caseIds: string[]): GatewayBenchCase[] {
  return resolveGatewayBenchCases(caseIds, GATEWAY_CASES, {
    allByDefault: true,
    validateDuplicatesFirst: true,
  });
}

function parseOptions(argv: string[] = process.argv.slice(2)): CliOptions {
  validateCliArgs(argv);
  return {
    cases: resolveCases(parseRepeatableFlag(argv, "--case")),
    cpuProfDir: parseFlagValue(argv, "--cpu-prof-dir"),
    entry: resolveEntry(parseFlagValue(argv, "--entry")),
    heapProfDir: parseFlagValue(argv, "--heap-prof-dir"),
    json: hasFlag(argv, "--json"),
    output: resolveOutputPath(parseFlagValue(argv, "--output")),
    runs: parsePositiveInt(parseFlagValue(argv, "--runs"), DEFAULT_RUNS, "--runs"),
    timeoutMs: parsePositiveInt(
      parseFlagValue(argv, "--timeout-ms"),
      DEFAULT_TIMEOUT_MS,
      "--timeout-ms",
    ),
    warmup: parseNonNegativeInt(parseFlagValue(argv, "--warmup"), DEFAULT_WARMUP, "--warmup"),
  };
}

function printUsage(): void {
  console.log(`OpenClaw Gateway startup benchmark

Usage:
  pnpm test:startup:gateway -- [options]
  node --import tsx scripts/bench-gateway-startup.ts [options]

Options:
  --case <id>          Specific case id to run; repeatable
  --entry <path>       Gateway CLI entry file (default: ${DEFAULT_ENTRY})
  --runs <n>           Measured runs per case (default: ${DEFAULT_RUNS})
  --warmup <n>         Warmup runs per case (default: ${DEFAULT_WARMUP})
  --timeout-ms <ms>    Per-run timeout (default: ${DEFAULT_TIMEOUT_MS})
  --cpu-prof-dir <dir> Write one V8 CPU profile per run
  --heap-prof-dir <dir> Write one V8 heap profile per run
  --output <path>      Write machine-readable JSON to a file
  --json               Emit machine-readable JSON
  --help, -h           Show this text

Case ids:
  ${GATEWAY_CASES.map((benchCase) => `${benchCase.id} (${benchCase.name})`).join("\n  ")}
`);
}

function summarizeCase(benchCase: GatewayBenchCase, samples: GatewaySample[]): CaseResult {
  const startupTraceKeys = new Set<string>();
  for (const sample of samples) {
    for (const key of Object.keys(sample.startupTrace)) {
      startupTraceKeys.add(key);
    }
  }
  const startupTrace: Record<string, SummaryStats> = {};
  for (const key of [...startupTraceKeys].toSorted()) {
    const stats = summarizeNumbers(
      samples
        .map((sample) => sample.startupTrace[key])
        .filter((value): value is number => typeof value === "number"),
    );
    if (stats) {
      startupTrace[key] = stats;
    }
  }
  return {
    id: benchCase.id,
    name: benchCase.name,
    samples,
    summary: {
      completionMs: summarizeNumbers(
        samples
          .map((sample) => sample.completionMs)
          .filter((value): value is number => typeof value === "number"),
      ),
      firstOutputMs: summarizeNumbers(
        samples
          .map((sample) => sample.firstOutputMs)
          .filter((value): value is number => typeof value === "number"),
      ),
      cpuCoreRatio: summarizeNumbers(
        samples
          .map((sample) => sample.cpuCoreRatio)
          .filter((value): value is number => typeof value === "number"),
      ),
      cpuMs: summarizeNumbers(
        samples
          .map((sample) => sample.cpuMs)
          .filter((value): value is number => typeof value === "number"),
      ),
      gatewayReadyLogMs: summarizeNumbers(
        samples
          .map((sample) => sample.gatewayReadyLogMs)
          .filter((value): value is number => typeof value === "number"),
      ),
      healthzMs: summarizeNumbers(
        samples
          .map((sample) => sample.healthz.ms)
          .filter((value): value is number => typeof value === "number"),
      ),
      httpListenLogMs: summarizeNumbers(
        samples
          .map((sample) => sample.httpListenLogMs)
          .filter((value): value is number => typeof value === "number"),
      ),
      maxRssMb: summarizeNumbers(
        samples
          .map((sample) => sample.maxRssMb)
          .filter((value): value is number => typeof value === "number"),
      ),
      readyzMs: summarizeNumbers(
        samples
          .map((sample) => sample.readyz.ms)
          .filter((value): value is number => typeof value === "number"),
      ),
      startupTrace,
    },
  };
}

function collectResultFailures(
  results: CaseResult[],
  options: { processMetricsRequired?: boolean } = {},
): BenchmarkFailure[] {
  const processMetricsRequired = options.processMetricsRequired ?? process.platform !== "win32";
  const failures: BenchmarkFailure[] = [];
  for (const result of results) {
    result.samples.forEach((sample, index) => {
      const missing: string[] = [];
      if (sample.healthz.status !== 200 || sample.healthz.ms == null) {
        missing.push("/healthz");
      }
      if (sample.readyz.status !== 200 || sample.readyz.ms == null) {
        missing.push("/readyz");
      }
      if (sample.completionMs == null) {
        missing.push("completion");
      }
      if (processMetricsRequired) {
        if (sample.cpuMs == null || sample.cpuCoreRatio == null) {
          missing.push("cpu");
        }
        if (sample.maxRssMb == null) {
          missing.push("rss");
        }
      }
      if (missing.length > 0) {
        failures.push({
          id: result.id,
          reason: `missing ${missing.join(", ")}`,
          sampleIndex: index + 1,
        });
        return;
      }
      if (sample.exitedBeforeTeardown === true) {
        failures.push({
          id: result.id,
          reason:
            sample.signal == null
              ? `child exited ${sample.exitCode ?? "before teardown"}`
              : `child exited by ${sample.signal}`,
          sampleIndex: index + 1,
        });
      }
    });
  }
  return failures;
}

function printBenchmarkFailures(failures: BenchmarkFailure[]): void {
  if (failures.length === 0) {
    return;
  }
  console.error(
    `[gateway-startup-bench] failed: ${failures.length} sample(s) did not produce ready probes or process metrics`,
  );
  for (const failure of failures.slice(0, 8)) {
    console.error(
      `[gateway-startup-bench] ${failure.id} run ${failure.sampleIndex}: ${failure.reason}`,
    );
  }
  if (failures.length > 8) {
    console.error(`[gateway-startup-bench] ${failures.length - 8} more sample failure(s) omitted`);
  }
}

function formatRatio(value: number | null): string {
  if (value == null) {
    return "n/a";
  }
  return value.toFixed(3);
}

function formatMemoryStats(stats: SummaryStats | null | undefined): string {
  if (!stats) {
    return "n/a";
  }
  return `p50=${formatMb(stats.p50)} avg=${formatMb(stats.avg)} min=${formatMb(stats.min)} max=${formatMb(stats.max)}`;
}

function formatRatioStats(stats: SummaryStats | null): string {
  if (!stats) {
    return "n/a";
  }
  return `p50=${formatRatio(stats.p50)} avg=${formatRatio(stats.avg)} min=${formatRatio(stats.min)} max=${formatRatio(stats.max)}`;
}

async function waitForStartupTracePhase(params: {
  deadlineAt: number;
  isDone: () => boolean;
  phase: string;
  startupTrace: Record<string, number>;
}): Promise<number | null> {
  const totalKey = `${params.phase}.total`;
  while (performance.now() < params.deadlineAt) {
    if (Object.hasOwn(params.startupTrace, totalKey)) {
      return params.startupTrace[totalKey] ?? null;
    }
    if (params.isDone()) {
      return null;
    }
    await delay(25);
  }
  return null;
}

function buildBenchAgentList(
  root: string,
  topology: GatewayBenchCase["agentTopology"],
): Array<{ id: string; default?: boolean; workspace: string }> | undefined {
  if (!topology) {
    return undefined;
  }
  const sharedWorkspace = path.join(root, "shared-workspace");
  const distinctWorkspace = path.join(root, "distinct-workspace");
  mkdirSync(sharedWorkspace, { recursive: true });
  if (topology === "single") {
    return [{ id: "main", default: true, workspace: sharedWorkspace }];
  }
  mkdirSync(distinctWorkspace, { recursive: true });
  return Array.from({ length: 12 }, (_, index) => ({
    id: `agent-${String(index + 1).padStart(2, "0")}`,
    ...(index === 0 ? { default: true } : {}),
    workspace: index === 11 ? distinctWorkspace : sharedWorkspace,
  }));
}

function writeConfig(root: string, benchCase: GatewayBenchCase): string {
  const hasCatalogFixture =
    benchCase.providerCatalogStallMs !== undefined ||
    benchCase.providerStaticCatalogStallMs !== undefined;
  const pluginCount = hasCatalogFixture ? 1 : benchCase.pluginCount;
  const pluginFixtures = pluginCount
    ? writePluginFixtures(root, {
        activationOnStartup: hasCatalogFixture ? true : benchCase.pluginActivationOnStartup,
        count: pluginCount,
        providerCatalogStallMs: benchCase.providerCatalogStallMs,
        providerStaticCatalogModelCount: benchCase.providerStaticCatalogModelCount,
        providerStaticCatalogStallMs: benchCase.providerStaticCatalogStallMs,
      })
    : null;
  const agentList = buildBenchAgentList(root, benchCase.agentTopology);
  return writeGatewayBenchConfig(root, benchCase.config, { agentList, pluginFixtures });
}

function sanitizedEnv(
  root: string,
  configPath: string,
  benchCase: GatewayBenchCase,
): NodeJS.ProcessEnv {
  return createGatewayBenchEnv(root, configPath, { caseEnv: benchCase.env });
}

function collectStartupTrace(line: string, startupTrace: Record<string, number>): void {
  collectTraceLine(line, "startup trace", startupTrace);
}

async function runGatewaySample(options: {
  benchCase: GatewayBenchCase;
  cpuProfDir?: string;
  entry: string;
  heapProfDir?: string;
  sampleIndex: number;
  timeoutMs: number;
}): Promise<GatewaySample> {
  const root = mkdtempSync(path.join(tmpdir(), "openclaw-gateway-bench-"));
  const port = await getFreePort();
  const configPath = writeConfig(root, options.benchCase);
  const env = sanitizedEnv(root, configPath, options.benchCase);
  const startAt = performance.now();
  const deadlineAt = startAt + options.timeoutMs;
  const startupTrace: Record<string, number> = {};
  const output: string[] = [];
  const outputBuffers: Record<"stderr" | "stdout", string> = { stderr: "", stdout: "" };
  let firstOutputMs: number | null = null;
  let gatewayReadyLogLine: string | null = null;
  let gatewayReadyLogMs: number | null = null;
  let httpListenLogLine: string | null = null;
  let httpListenLogMs: number | null = null;
  let maxRssMb: number | null = null;
  let childExited = false;

  const nodeOptions = [
    ...(options.cpuProfDir
      ? [
          "--cpu-prof",
          "--cpu-prof-dir",
          options.cpuProfDir,
          "--cpu-prof-name",
          `openclaw-gateway-${options.benchCase.id}-${options.sampleIndex}-${Date.now()}.cpuprofile`,
        ]
      : []),
    ...(options.heapProfDir ? ["--heap-prof", "--heap-prof-dir", options.heapProfDir] : []),
  ];
  const childArgs = buildGatewayBenchChildArgs(options.entry, port, nodeOptions);
  const child = spawn(process.execPath, childArgs, {
    cwd: process.cwd(),
    detached: process.platform !== "win32",
    env,
  });
  const cpuStartMs = readProcessTreeCpuMs(child.pid);
  const sampleRss = () => {
    const rssMb = readProcessRssMb(child.pid);
    if (rssMb != null) {
      maxRssMb = maxRssMb == null ? rssMb : Math.max(maxRssMb, rssMb);
    }
  };
  sampleRss();
  const rssTimer = setInterval(sampleRss, 100);
  rssTimer.unref?.();
  const childExitPromise = new Promise<{ exitCode: number | null; signal: string | null }>(
    (resolve) => {
      child.once("exit", (exitCode, signal) => {
        childExited = true;
        resolve({ exitCode, signal });
      });
    },
  );

  const onLine = (line: string, nowMs: number) => {
    const readyLogKind = classifyGatewayReadyLog(line);
    if (readyLogKind === "http-listen" && httpListenLogMs == null) {
      httpListenLogMs = nowMs;
      httpListenLogLine = line;
    }
    if (readyLogKind === "gateway-ready" && gatewayReadyLogMs == null) {
      gatewayReadyLogMs = nowMs;
      gatewayReadyLogLine = line;
    }
    collectStartupTrace(line, startupTrace);
  };
  const onChunk = (stream: "stderr" | "stdout", chunk: Buffer) => {
    if (firstOutputMs == null) {
      firstOutputMs = performance.now() - startAt;
    }
    const text = chunk.toString("utf8");
    output.push(text);
    if (output.length > 20) {
      output.splice(0, output.length - 20);
    }
    const parsed = collectOutputLines(outputBuffers[stream], text);
    outputBuffers[stream] = parsed.carry;
    const nowMs = performance.now() - startAt;
    for (const line of parsed.lines) {
      onLine(line, nowMs);
    }
  };
  child.stdout.on("data", (chunk: Buffer) => onChunk("stdout", chunk));
  child.stderr.on("data", (chunk: Buffer) => onChunk("stderr", chunk));

  const [healthz, readyz] = await Promise.all([
    waitForProbe({
      deadlineAt,
      isDone: () => childExited,
      path: "/healthz",
      port,
      startAt,
    }),
    waitForProbe({
      deadlineAt,
      isDone: () => childExited,
      path: "/readyz",
      port,
      startAt,
    }),
  ]);
  const completionMs = options.benchCase.completionTracePhase
    ? await waitForStartupTracePhase({
        deadlineAt,
        isDone: () => childExited,
        phase: options.benchCase.completionTracePhase,
        startupTrace,
      })
    : performance.now() - startAt;
  const completedAt = performance.now();
  const cpuEndMs = readProcessTreeCpuMs(child.pid);
  const cpuMs = cpuStartMs == null || cpuEndMs == null ? null : Math.max(0, cpuEndMs - cpuStartMs);
  const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, completedAt - startAt);
  const exit = await stopChild(child);
  clearInterval(rssTimer);
  sampleRss();
  // stopChild is the bounded teardown wait; the raw exit promise may never settle.
  void childExitPromise.catch(() => null);
  flushOutputLineBuffers(outputBuffers, onLine, performance.now() - startAt, {
    flushPartial: true,
  });
  rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 });

  return {
    completionMs,
    cpuCoreRatio,
    cpuMs,
    exitedBeforeTeardown: exit.exitedBeforeTeardown,
    exitCode: exit.exitCode,
    firstOutputMs,
    gatewayReadyLogLine,
    gatewayReadyLogMs,
    healthz,
    httpListenLogLine,
    httpListenLogMs,
    maxRssMb,
    outputTail: output.join("").split(/\r?\n/u).slice(-20).join("\n"),
    readyz,
    signal: exit.signal,
    startupTrace,
  };
}

async function runCase(options: {
  benchCase: GatewayBenchCase;
  cpuProfDir?: string;
  entry: string;
  heapProfDir?: string;
  runs: number;
  timeoutMs: number;
  warmup: number;
}): Promise<CaseResult> {
  const samples: GatewaySample[] = [];
  const total = options.runs + options.warmup;
  for (let index = 0; index < total; index += 1) {
    const sample = await runGatewaySample({
      benchCase: options.benchCase,
      cpuProfDir: options.cpuProfDir,
      entry: options.entry,
      heapProfDir: options.heapProfDir,
      sampleIndex: index + 1,
      timeoutMs: options.timeoutMs,
    });
    if (index >= options.warmup) {
      samples.push(sample);
      const heapUsedMb = sample.startupTrace["memory.ready.heapUsedMb"] ?? null;
      console.log(
        `[gateway-startup-bench] ${options.benchCase.id} run ${samples.length}/${options.runs}: completion=${formatMs(sample.completionMs)} healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} httpListen=${formatMs(sample.httpListenLogMs)} gatewayReady=${formatMs(sample.gatewayReadyLogMs)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)} heap=${formatMb(heapUsedMb)}`,
      );
      if (
        sample.outputTail &&
        (sample.completionMs == null ||
          sample.healthz.status !== 200 ||
          sample.readyz.status !== 200)
      ) {
        console.error(
          `[gateway-startup-bench] ${options.benchCase.id} output tail:\n${sample.outputTail}`,
        );
      }
    } else {
      const heapUsedMb = sample.startupTrace["memory.ready.heapUsedMb"] ?? null;
      console.log(
        `[gateway-startup-bench] ${options.benchCase.id} warmup ${index + 1}/${options.warmup}: healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)} heap=${formatMb(heapUsedMb)}`,
      );
    }
  }
  return summarizeCase(options.benchCase, samples);
}

function printResult(result: CaseResult): void {
  console.log(`\n${result.name} (${result.id})`);
  console.log(`  completion:   ${formatStats(result.summary.completionMs)}`);
  console.log(`  first output: ${formatStats(result.summary.firstOutputMs)}`);
  console.log(`  CPU:          ${formatStats(result.summary.cpuMs)}`);
  console.log(`  CPU core:     ${formatRatioStats(result.summary.cpuCoreRatio)}`);
  console.log(`  /healthz:     ${formatStats(result.summary.healthzMs)}`);
  console.log(`  http listen:  ${formatStats(result.summary.httpListenLogMs)}`);
  console.log(`  gateway ready: ${formatStats(result.summary.gatewayReadyLogMs)}`);
  console.log(`  /readyz:      ${formatStats(result.summary.readyzMs)}`);
  console.log(`  max RSS:      ${formatMemoryStats(result.summary.maxRssMb)}`);
  console.log(
    `  ready memory: rss=${formatMemoryStats(result.summary.startupTrace["memory.ready.rssMb"])} heap=${formatMemoryStats(result.summary.startupTrace["memory.ready.heapUsedMb"])} external=${formatMemoryStats(result.summary.startupTrace["memory.ready.externalMb"])}`,
  );
  console.log(
    `  post-ready memory: rss=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.rssMb"])} heap=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.heapUsedMb"])} external=${formatMemoryStats(result.summary.startupTrace["memory.post-ready.externalMb"])}`,
  );
  console.log(
    `  prepared runtime: agents=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.agentCount"])} workspaces=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.workspaceGroupCount"])} configuredGroups=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredFactsGroupCount"])} configuredModels=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredRuntimeModelCount"])} generatedPlugins=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.generatedCatalogPluginCount"])} generatedReads=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.generatedCatalogReadCount"])} sources=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogSourceCount"])} credentials=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.credentialGroupCount"])} catalogs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogGroupCount"])} registries=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.runtimeRegistryCount"])} workspaceFacts=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.workspaceFactsMs"])} runtimePlugins=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.runtimePluginMs"])} metadata=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.pluginMetadataMs"])} staticProviders=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.staticProviderCatalogMs"])} ambientAuth=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.ambientCredentialsMs"])} agentFacts=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.agentFactsMs"])} configuredProjection=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredProjectionMs"])} sourceMs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogSourceMs"])} registryMs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.registryMs"])} sourceLimit=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.sourceConcurrencyLimitCount"])} fullCatalogLimit=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.fullCatalogConcurrencyLimitCount"])} staticCatalog=${formatStats(result.summary.startupTrace["benchmark.preparedRuntimeStaticCatalogCallCount"])} pluginLoader=${formatStats(result.summary.startupTrace["sidecars.plugin-loader.callsCount"])} eventLoopMax=${formatStats(result.summary.startupTrace["sidecars.model-runtime.eventLoopMax"])}`,
  );
  const trace = selectSlowStartupTraceDurations(result.summary.startupTrace, 8);
  if (trace.length > 0) {
    console.log("  trace top:");
    for (const [name, stats] of trace) {
      console.log(`    ${name}: ${formatStats(stats)}`);
    }
  }
}

async function main() {
  const argv = process.argv.slice(2);
  if (hasHelpFlag(argv)) {
    printUsage();
    return;
  }

  const options = parseOptions(argv);
  if (options.cpuProfDir) {
    mkdirSync(options.cpuProfDir, { recursive: true });
  }
  if (options.heapProfDir) {
    mkdirSync(options.heapProfDir, { recursive: true });
  }
  const results: CaseResult[] = [];
  for (const benchCase of options.cases) {
    results.push(
      await runCase({
        benchCase,
        cpuProfDir: options.cpuProfDir,
        entry: options.entry,
        heapProfDir: options.heapProfDir,
        runs: options.runs,
        timeoutMs: options.timeoutMs,
        warmup: options.warmup,
      }),
    );
  }

  const payload = {
    entry: options.entry,
    generatedAt: new Date().toISOString(),
    results,
  };
  if (options.output) {
    mkdirSync(path.dirname(options.output), { recursive: true });
    writeFileSync(options.output, `${JSON.stringify(payload, null, 2)}\n`);
  }
  if (options.json) {
    console.log(JSON.stringify(payload, null, 2));
  } else {
    for (const result of results) {
      printResult(result);
    }
  }

  const failures = collectResultFailures(results);
  if (failures.length > 0) {
    printBenchmarkFailures(failures);
    process.exitCode = 1;
  }
}

export const testing = {
  classifyGatewayReadyLog,
  collectOutputLines,
  collectResultFailures,
  collectStartupTrace,
  parseOptions,
  parseNonNegativeInt,
  parsePositiveInt,
  parseProcessRssKb,
  resolveEntry,
  sanitizedEnv,
  stopChild,
  summarizeCase,
  validateCliArgs,
  waitForProbe,
  waitForStartupTracePhase,
  writeConfig,
};

if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
  main().catch((err: unknown) => {
    if (err instanceof CliArgumentError) {
      console.error(err.message);
      process.exitCode = 1;
      return;
    }
    console.error(err instanceof Error ? err.stack : String(err));
    process.exitCode = 1;
  });
}