From 825a79467255de027a5215b7942f17ab2eff50cf Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 2 Aug 2026 07:43:22 +0800 Subject: [PATCH] fix(perf): retain startup benchmark warmups (#117649) --- scripts/bench-cli-startup.ts | 68 ++++++++++++------- test/scripts/bench-cli-startup.test.ts | 35 ++++++++++ .../scripts/cli-startup-bench-spawner.test.ts | 21 +++++- 3 files changed, 96 insertions(+), 28 deletions(-) diff --git a/scripts/bench-cli-startup.ts b/scripts/bench-cli-startup.ts index 349849d8b149..47fe8c6a006b 100644 --- a/scripts/bench-cli-startup.ts +++ b/scripts/bench-cli-startup.ts @@ -25,11 +25,18 @@ type Sample = { maxRssMb: number | null; exitCode: number | null; signal: string | null; + startedAt?: string; + endedAt?: string; timedOut?: boolean; stdoutTail?: string; stderrTail?: string; }; +type CaseRuns = { + warmupSamples: Sample[]; + samples: Sample[]; +}; + type SummaryStats = { avg: number; p50: number; @@ -58,6 +65,7 @@ type SuiteResult = { firstOutputBudgetMs: number | null; exitBudgetMs: number | null; } | null; + warmupSamples?: Sample[]; samples: Sample[]; summary: CaseSummary; }>; @@ -754,6 +762,7 @@ async function runSample(params: { params.entry, ...params.commandCase.args, ]; + const startedAt = new Date(); const started = process.hrtime.bigint(); let firstOutputMs: number | null = null; let stdout = ""; @@ -798,6 +807,8 @@ async function runSample(params: { ms, firstOutputMs, maxRssMb: parseMaxRssMb(stderr), + startedAt: startedAt.toISOString(), + endedAt: new Date().toISOString(), ...(timedOut ? { timedOut } : {}), ...sample, }); @@ -956,7 +967,8 @@ async function runCase(params: { cpuProfDir?: string; heapProfDir?: string; rssHookPath: string; -}): Promise { +}): Promise { + const warmupSamples: Sample[] = []; const samples: Sample[] = []; const totalRuns = params.warmup + params.runs; const caseRunRoot = @@ -967,11 +979,12 @@ async function runCase(params: { for (let i = 0; i < totalRuns; i += 1) { const sample = await runSample({ ...params, runRoot: caseRunRoot }); if (i < params.warmup) { + warmupSamples.push(sample); continue; } samples.push(sample); } - return samples; + return { warmupSamples, samples }; } finally { if (caseRunRoot) { rmSync(caseRunRoot, { recursive: true, force: true }); @@ -1064,30 +1077,34 @@ export function collectFailedSamples(result: SuiteResult): string[] { for (const commandCase of result.cases) { if (commandCase.samples.length === 0) { failures.push(`${result.entry} ${commandCase.id}: no measured samples`); - continue; } - for (const [sampleIndex, sample] of commandCase.samples.entries()) { - const label = `${result.entry} ${commandCase.id} sample ${sampleIndex + 1}`; - const expectedExitCodes = new Set(commandCase.expectedExitCodes ?? [0]); - if (sample.timedOut === true) { - failures.push(`${label}: timed out`); - } else if (sample.signal !== null) { - failures.push(`${label}: exited via signal ${sample.signal}`); - } else if (!expectedExitCodes.has(sample.exitCode ?? -1)) { - failures.push(`${label}: exited with code ${String(sample.exitCode)}`); - } else if (sample.maxRssMb === null) { - failures.push(`${label}: did not report max RSS`); - } else if (sample.exitCode !== 0) { - const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`; - const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter( - (snippet) => !output.includes(snippet), - ); - if (missing.length > 0) { - failures.push( - `${label}: exited with expected code ${String( - sample.exitCode, - )} but output did not match expected clean-state markers (${missing.join(", ")})`, + for (const [sampleKind, samples] of [ + ["warmup", commandCase.warmupSamples ?? []], + ["sample", commandCase.samples], + ] as const) { + for (const [sampleIndex, sample] of samples.entries()) { + const label = `${result.entry} ${commandCase.id} ${sampleKind} ${sampleIndex + 1}`; + const expectedExitCodes = new Set(commandCase.expectedExitCodes ?? [0]); + if (sample.timedOut === true) { + failures.push(`${label}: timed out`); + } else if (sample.signal !== null) { + failures.push(`${label}: exited via signal ${sample.signal}`); + } else if (!expectedExitCodes.has(sample.exitCode ?? -1)) { + failures.push(`${label}: exited with code ${String(sample.exitCode)}`); + } else if (sample.maxRssMb === null) { + failures.push(`${label}: did not report max RSS`); + } else if (sample.exitCode !== 0) { + const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`; + const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter( + (snippet) => !output.includes(snippet), ); + if (missing.length > 0) { + failures.push( + `${label}: exited with expected code ${String( + sample.exitCode, + )} but output did not match expected clean-state markers (${missing.join(", ")})`, + ); + } } } } @@ -1102,7 +1119,7 @@ async function buildSuiteResult(params: { }): Promise { const cases = []; for (const commandCase of params.options.cases) { - const samples = await runCase({ + const { warmupSamples, samples } = await runCase({ entry: params.entry, commandCase, runs: params.options.runs, @@ -1129,6 +1146,7 @@ async function buildSuiteResult(params: { exitBudgetMs: commandCase.exitBudgetMs ?? null, } : null, + warmupSamples, samples, summary: summarizeSamples(samples), }); diff --git a/test/scripts/bench-cli-startup.test.ts b/test/scripts/bench-cli-startup.test.ts index 506d76812331..55aa58aca67f 100644 --- a/test/scripts/bench-cli-startup.test.ts +++ b/test/scripts/bench-cli-startup.test.ts @@ -310,6 +310,41 @@ describe("bench-cli-startup", () => { ]); }); + it("retains and validates warmup samples separately from measured samples", () => { + const passingSample = { + ms: 10, + firstOutputMs: 5, + maxRssMb: 50, + exitCode: 0, + signal: null, + startedAt: "2026-08-01T20:00:00.000Z", + endedAt: "2026-08-01T20:00:00.010Z", + }; + + expect( + testing.collectFailedSamples({ + entry: "dist/entry.js", + cases: [ + { + id: "gatewayHealthJsonConnected", + name: "gateway health --json (connected)", + args: ["gateway", "health", "--json"], + contract: null, + warmupSamples: [{ ...passingSample, exitCode: 1 }], + samples: [passingSample], + summary: { + sampleCount: 1, + durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 }, + firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 }, + maxRssMb: { avg: 50, p50: 50, p95: 50, min: 50, max: 50 }, + exitSummary: "code:0x1", + }, + }, + ], + }), + ).toEqual(["dist/entry.js gatewayHealthJsonConnected warmup 1: exited with code 1"]); + }); + it("fails reports with samples that did not report RSS", () => { expect( testing.collectFailedSamples({ diff --git a/test/scripts/cli-startup-bench-spawner.test.ts b/test/scripts/cli-startup-bench-spawner.test.ts index 50a84d997b61..9264c10441b1 100644 --- a/test/scripts/cli-startup-bench-spawner.test.ts +++ b/test/scripts/cli-startup-bench-spawner.test.ts @@ -51,6 +51,7 @@ describe("CLI startup benchmark script spawners", () => { const runCase = (caseId: string) => { fs.rmSync(homeLogPath, { force: true }); + const reportPath = path.join(tmpDir, `${caseId}.json`); execFileSync( process.execPath, [ @@ -65,6 +66,8 @@ describe("CLI startup benchmark script spawners", () => { "2", "--warmup", "1", + "--output", + reportPath, ], { cwd: process.cwd(), @@ -75,16 +78,28 @@ describe("CLI startup benchmark script spawners", () => { stdio: "pipe", }, ); - return fs.readFileSync(homeLogPath, "utf8").trim().split("\n"); + return { + homes: fs.readFileSync(homeLogPath, "utf8").trim().split("\n"), + report: JSON.parse(fs.readFileSync(reportPath, "utf8")), + }; }; - const warmedHomes = runCase("gatewayHealthJsonConnected"); + const warmed = runCase("gatewayHealthJsonConnected"); + const warmedHomes = warmed.homes; expect(warmedHomes).toHaveLength(3); expect(new Set(warmedHomes).size).toBe(1); expect(warmedHomes.every((home) => !fs.existsSync(home))).toBe(true); + const warmedCase = warmed.report.primary.cases[0]; + expect(warmedCase.warmupSamples).toHaveLength(1); + expect(warmedCase.samples).toHaveLength(2); + for (const sample of [...warmedCase.warmupSamples, ...warmedCase.samples]) { + expect(new Date(sample.startedAt).toISOString()).toBe(sample.startedAt); + expect(new Date(sample.endedAt).toISOString()).toBe(sample.endedAt); + expect(Date.parse(sample.endedAt)).toBeGreaterThanOrEqual(Date.parse(sample.startedAt)); + } for (const caseId of ["gatewayHealthJson", "gatewayHealthJsonFirstDevice"]) { - const sampleHomes = runCase(caseId); + const sampleHomes = runCase(caseId).homes; expect(sampleHomes).toHaveLength(3); expect(new Set(sampleHomes).size).toBe(3); expect(sampleHomes.every((home) => !fs.existsSync(home))).toBe(true);