diff --git a/scripts/bench-cli-startup.ts b/scripts/bench-cli-startup.ts index d763a81d5fc2..11fd4781d731 100644 --- a/scripts/bench-cli-startup.ts +++ b/scripts/bench-cli-startup.ts @@ -838,6 +838,8 @@ export function collectFailedSamples(result: SuiteResult): string[] { failures.push(`${label}: exited via signal ${sample.signal}`); } else if (!expectedExitCodes.has(sample.exitCode ?? -1)) { failures.push(`${label}: exited with code ${String(sample.exitCode)}`); + } else if (sample.maxRssMb === null) { + failures.push(`${label}: did not report max RSS`); } else if (sample.exitCode !== 0) { const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`; const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter( diff --git a/scripts/test-cli-startup-bench-budget.mjs b/scripts/test-cli-startup-bench-budget.mjs index 035b305deb9a..e396d9c58ddb 100644 --- a/scripts/test-cli-startup-bench-budget.mjs +++ b/scripts/test-cli-startup-bench-budget.mjs @@ -174,6 +174,10 @@ for (const currentCase of currentCases.values()) { console.error(`[test-cli-startup-bench-budget] ${currentCase.name} timed out.`); failed = true; } + if (samples.some((sample) => !Number.isFinite(sample.maxRssMb))) { + console.error(`[test-cli-startup-bench-budget] ${currentCase.name} did not report max RSS.`); + failed = true; + } } if (!opts.skipBaseline) { diff --git a/test/scripts/bench-cli-startup.test.ts b/test/scripts/bench-cli-startup.test.ts index fdc551c38311..d073a98e10d0 100644 --- a/test/scripts/bench-cli-startup.test.ts +++ b/test/scripts/bench-cli-startup.test.ts @@ -91,6 +91,38 @@ describe("bench-cli-startup", () => { ]); }); + it("fails reports with samples that did not report RSS", () => { + expect( + testing.collectFailedSamples({ + entry: "openclaw.mjs", + cases: [ + { + id: "version", + name: "--version", + args: ["--version"], + contract: null, + samples: [ + { + ms: 10, + firstOutputMs: 5, + maxRssMb: null, + exitCode: 0, + signal: null, + }, + ], + summary: { + sampleCount: 1, + durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 }, + firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 }, + maxRssMb: null, + exitSummary: "code:0x1", + }, + }, + ], + }), + ).toEqual(["openclaw.mjs version sample 1: did not report max RSS"]); + }); + it("allows declared nonzero exit codes for clean-state probes", () => { const sample = { ms: 10, diff --git a/test/scripts/cli-startup-bench-spawner.test.ts b/test/scripts/cli-startup-bench-spawner.test.ts index b9a5e4889fc7..23545a8dbfb5 100644 --- a/test/scripts/cli-startup-bench-spawner.test.ts +++ b/test/scripts/cli-startup-bench-spawner.test.ts @@ -270,6 +270,46 @@ describe("CLI startup benchmark script spawners", () => { } }); + it("fails reused reports with missing RSS samples", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-rss-test-")); + try { + const baselinePath = path.join(tmpDir, "baseline.json"); + const reportPath = path.join(tmpDir, "current.json"); + const missingRssCase = { + id: "version", + name: "--version", + samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: null, exitCode: 0, signal: null }], + summary: { + durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 }, + firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 }, + maxRssMb: null, + }, + }; + fs.writeFileSync(baselinePath, JSON.stringify({ primary: { cases: [missingRssCase] } })); + fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [missingRssCase] } })); + + const result = spawnSync( + process.execPath, + [ + "scripts/test-cli-startup-bench-budget.mjs", + "--baseline", + baselinePath, + "--report", + reportPath, + "--skip-baseline", + ], + { cwd: process.cwd(), encoding: "utf8" }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "[test-cli-startup-bench-budget] --version did not report max RSS.", + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + it("rejects malformed startup budget env vars before reading reports", () => { const result = spawnSync(process.execPath, ["scripts/test-cli-startup-bench-budget.mjs"], { cwd: process.cwd(),