fix(test): fail startup bench missing RSS samples

This commit is contained in:
Vincent Koc
2026-06-06 20:39:07 +02:00
parent a1ffaafc12
commit 77f8b16716
4 changed files with 78 additions and 0 deletions
+2
View File
@@ -838,6 +838,8 @@ export function collectFailedSamples(result: SuiteResult): string[] {
failures.push(`${label}: exited via signal ${sample.signal}`);
} else if (!expectedExitCodes.has(sample.exitCode ?? -1)) {
failures.push(`${label}: exited with code ${String(sample.exitCode)}`);
} else if (sample.maxRssMb === null) {
failures.push(`${label}: did not report max RSS`);
} else if (sample.exitCode !== 0) {
const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`;
const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter(
@@ -174,6 +174,10 @@ for (const currentCase of currentCases.values()) {
console.error(`[test-cli-startup-bench-budget] ${currentCase.name} timed out.`);
failed = true;
}
if (samples.some((sample) => !Number.isFinite(sample.maxRssMb))) {
console.error(`[test-cli-startup-bench-budget] ${currentCase.name} did not report max RSS.`);
failed = true;
}
}
if (!opts.skipBaseline) {
+32
View File
@@ -91,6 +91,38 @@ describe("bench-cli-startup", () => {
]);
});
it("fails reports with samples that did not report RSS", () => {
expect(
testing.collectFailedSamples({
entry: "openclaw.mjs",
cases: [
{
id: "version",
name: "--version",
args: ["--version"],
contract: null,
samples: [
{
ms: 10,
firstOutputMs: 5,
maxRssMb: null,
exitCode: 0,
signal: null,
},
],
summary: {
sampleCount: 1,
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
maxRssMb: null,
exitSummary: "code:0x1",
},
},
],
}),
).toEqual(["openclaw.mjs version sample 1: did not report max RSS"]);
});
it("allows declared nonzero exit codes for clean-state probes", () => {
const sample = {
ms: 10,
@@ -270,6 +270,46 @@ describe("CLI startup benchmark script spawners", () => {
}
});
it("fails reused reports with missing RSS samples", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-rss-test-"));
try {
const baselinePath = path.join(tmpDir, "baseline.json");
const reportPath = path.join(tmpDir, "current.json");
const missingRssCase = {
id: "version",
name: "--version",
samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: null, exitCode: 0, signal: null }],
summary: {
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
maxRssMb: null,
},
};
fs.writeFileSync(baselinePath, JSON.stringify({ primary: { cases: [missingRssCase] } }));
fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [missingRssCase] } }));
const result = spawnSync(
process.execPath,
[
"scripts/test-cli-startup-bench-budget.mjs",
"--baseline",
baselinePath,
"--report",
reportPath,
"--skip-baseline",
],
{ cwd: process.cwd(), encoding: "utf8" },
);
expect(result.status).toBe(1);
expect(result.stderr).toContain(
"[test-cli-startup-bench-budget] --version did not report max RSS.",
);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
it("rejects malformed startup budget env vars before reading reports", () => {
const result = spawnSync(process.execPath, ["scripts/test-cli-startup-bench-budget.mjs"], {
cwd: process.cwd(),