ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(perf): retain startup benchmark warmups (#117649)

Browse Source
This commit is contained in:
Vincent Koc
2026-08-02 07:43:22 +08:00
committed by GitHub
parent 2f06ed30d8
commit 825a794672
3 changed files with 96 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench-cli-startup.ts
+43 -25
View File
@@ -25,11 +25,18 @@ type Sample = {
maxRssMb: number | null;
exitCode: number | null;
signal: string | null;
startedAt?: string;
endedAt?: string;
timedOut?: boolean;
stdoutTail?: string;
stderrTail?: string;
};
type CaseRuns = {
warmupSamples: Sample[];
samples: Sample[];
};
type SummaryStats = {
avg: number;
p50: number;
@@ -58,6 +65,7 @@ type SuiteResult = {
firstOutputBudgetMs: number | null;
exitBudgetMs: number | null;
} | null;
warmupSamples?: Sample[];
samples: Sample[];
summary: CaseSummary;
}>;
@@ -754,6 +762,7 @@ async function runSample(params: {
params.entry,
...params.commandCase.args,
];
const startedAt = new Date();
const started = process.hrtime.bigint();
let firstOutputMs: number | null = null;
let stdout = "";
@@ -798,6 +807,8 @@ async function runSample(params: {
ms,
firstOutputMs,
maxRssMb: parseMaxRssMb(stderr),
startedAt: startedAt.toISOString(),
endedAt: new Date().toISOString(),
...(timedOut ? { timedOut } : {}),
...sample,
});
@@ -956,7 +967,8 @@ async function runCase(params: {
cpuProfDir?: string;
heapProfDir?: string;
rssHookPath: string;
}): Promise<Sample[]> {
}): Promise<CaseRuns> {
const warmupSamples: Sample[] = [];
const samples: Sample[] = [];
const totalRuns = params.warmup + params.runs;
const caseRunRoot =
@@ -967,11 +979,12 @@ async function runCase(params: {
for (let i = 0; i < totalRuns; i += 1) {
const sample = await runSample({ ...params, runRoot: caseRunRoot });
if (i < params.warmup) {
warmupSamples.push(sample);
continue;
}
samples.push(sample);
}
return samples;
return { warmupSamples, samples };
} finally {
if (caseRunRoot) {
rmSync(caseRunRoot, { recursive: true, force: true });
@@ -1064,30 +1077,34 @@ export function collectFailedSamples(result: SuiteResult): string[] {
for (const commandCase of result.cases) {
if (commandCase.samples.length === 0) {
failures.push(`${result.entry} ${commandCase.id}: no measured samples`);
continue;
}
for (const [sampleIndex, sample] of commandCase.samples.entries()) {
const label = `${result.entry} ${commandCase.id} sample ${sampleIndex + 1}`;
const expectedExitCodes = new Set(commandCase.expectedExitCodes ?? [0]);
if (sample.timedOut === true) {
failures.push(`${label}: timed out`);
} else if (sample.signal !== null) {
failures.push(`${label}: exited via signal ${sample.signal}`);
} else if (!expectedExitCodes.has(sample.exitCode ?? -1)) {
failures.push(`${label}: exited with code ${String(sample.exitCode)}`);
} else if (sample.maxRssMb === null) {
failures.push(`${label}: did not report max RSS`);
} else if (sample.exitCode !== 0) {
const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`;
const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter(
(snippet) => !output.includes(snippet),
);
if (missing.length > 0) {
failures.push(
`${label}: exited with expected code ${String(
sample.exitCode,
)} but output did not match expected clean-state markers (${missing.join(", ")})`,
for (const [sampleKind, samples] of [
["warmup", commandCase.warmupSamples ?? []],
["sample", commandCase.samples],
] as const) {
for (const [sampleIndex, sample] of samples.entries()) {
const label = `${result.entry} ${commandCase.id} ${sampleKind} ${sampleIndex + 1}`;
const expectedExitCodes = new Set(commandCase.expectedExitCodes ?? [0]);
if (sample.timedOut === true) {
failures.push(`${label}: timed out`);
} else if (sample.signal !== null) {
failures.push(`${label}: exited via signal ${sample.signal}`);
} else if (!expectedExitCodes.has(sample.exitCode ?? -1)) {
failures.push(`${label}: exited with code ${String(sample.exitCode)}`);
} else if (sample.maxRssMb === null) {
failures.push(`${label}: did not report max RSS`);
} else if (sample.exitCode !== 0) {
const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`;
const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter(
(snippet) => !output.includes(snippet),
);
if (missing.length > 0) {
failures.push(
`${label}: exited with expected code ${String(
sample.exitCode,
)} but output did not match expected clean-state markers (${missing.join(", ")})`,
);
}
}
}
}
@@ -1102,7 +1119,7 @@ async function buildSuiteResult(params: {
}): Promise<SuiteResult> {
const cases = [];
for (const commandCase of params.options.cases) {
const samples = await runCase({
const { warmupSamples, samples } = await runCase({
entry: params.entry,
commandCase,
runs: params.options.runs,
@@ -1129,6 +1146,7 @@ async function buildSuiteResult(params: {
exitBudgetMs: commandCase.exitBudgetMs ?? null,
}
: null,
warmupSamples,
samples,
summary: summarizeSamples(samples),
});
test/scripts/bench-cli-startup.test.ts
+35
View File
@@ -310,6 +310,41 @@ describe("bench-cli-startup", () => {
]);
});
it("retains and validates warmup samples separately from measured samples", () => {
const passingSample = {
ms: 10,
firstOutputMs: 5,
maxRssMb: 50,
exitCode: 0,
signal: null,
startedAt: "2026-08-01T20:00:00.000Z",
endedAt: "2026-08-01T20:00:00.010Z",
};
expect(
testing.collectFailedSamples({
entry: "dist/entry.js",
cases: [
{
id: "gatewayHealthJsonConnected",
name: "gateway health --json (connected)",
args: ["gateway", "health", "--json"],
contract: null,
warmupSamples: [{ ...passingSample, exitCode: 1 }],
samples: [passingSample],
summary: {
sampleCount: 1,
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
maxRssMb: { avg: 50, p50: 50, p95: 50, min: 50, max: 50 },
exitSummary: "code:0x1",
},
},
],
}),
).toEqual(["dist/entry.js gatewayHealthJsonConnected warmup 1: exited with code 1"]);
});
it("fails reports with samples that did not report RSS", () => {
expect(
testing.collectFailedSamples({
test/scripts/cli-startup-bench-spawner.test.ts
+18 -3
View File
@@ -51,6 +51,7 @@ describe("CLI startup benchmark script spawners", () => {
const runCase = (caseId: string) => {
fs.rmSync(homeLogPath, { force: true });
const reportPath = path.join(tmpDir, `${caseId}.json`);
execFileSync(
process.execPath,
[
@@ -65,6 +66,8 @@ describe("CLI startup benchmark script spawners", () => {
"2",
"--warmup",
"1",
"--output",
reportPath,
],
{
cwd: process.cwd(),
@@ -75,16 +78,28 @@ describe("CLI startup benchmark script spawners", () => {
stdio: "pipe",
},
);
return fs.readFileSync(homeLogPath, "utf8").trim().split("\n");
return {
homes: fs.readFileSync(homeLogPath, "utf8").trim().split("\n"),
report: JSON.parse(fs.readFileSync(reportPath, "utf8")),
};
};
const warmedHomes = runCase("gatewayHealthJsonConnected");
const warmed = runCase("gatewayHealthJsonConnected");
const warmedHomes = warmed.homes;
expect(warmedHomes).toHaveLength(3);
expect(new Set(warmedHomes).size).toBe(1);
expect(warmedHomes.every((home) => !fs.existsSync(home))).toBe(true);
const warmedCase = warmed.report.primary.cases[0];
expect(warmedCase.warmupSamples).toHaveLength(1);
expect(warmedCase.samples).toHaveLength(2);
for (const sample of [...warmedCase.warmupSamples, ...warmedCase.samples]) {
expect(new Date(sample.startedAt).toISOString()).toBe(sample.startedAt);
expect(new Date(sample.endedAt).toISOString()).toBe(sample.endedAt);
expect(Date.parse(sample.endedAt)).toBeGreaterThanOrEqual(Date.parse(sample.startedAt));
}
for (const caseId of ["gatewayHealthJson", "gatewayHealthJsonFirstDevice"]) {
const sampleHomes = runCase(caseId);
const sampleHomes = runCase(caseId).homes;
expect(sampleHomes).toHaveLength(3);
expect(new Set(sampleHomes).size).toBe(3);
expect(sampleHomes.every((home) => !fs.existsSync(home))).toBe(true);