ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(perf): separate warm and first-device health probes (#117525)

Browse Source 
* fix(perf): separate warm and first-device health probes

* fix(perf): configure first-device health probe

* fix(perf): require connected health probes

* fix(perf): preserve generic health benchmark state
This commit is contained in:
Vincent Koc
2026-08-02 03:30:17 +08:00
committed by GitHub
parent 936c9c7e2d
commit 6531ca91f4
5 changed files with 178 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/openclaw-performance.yml
+2 -1
View File
@@ -807,7 +807,8 @@ jobs:
OPENCLAW_HOME="$gateway_home" OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" OPENCLAW_GATEWAY_PORT="$gateway_port" \
node --import tsx scripts/bench-cli-startup.ts \
--case gatewayHealthJson \
--case gatewayHealthJsonConnected \
--case gatewayHealthJsonFirstDevice \
--case configGetGatewayPort \
--runs "$source_runs" \
--warmup 1 \
scripts/bench-cli-startup.ts
+38 -8
View File
@@ -12,6 +12,7 @@ type CommandCase = {
name: string;
args: string[];
presets: readonly string[];
stateScope?: "case" | "sample";
expectedExitCodes?: readonly number[];
expectedNonzeroOutputIncludes?: readonly string[];
firstOutputBudgetMs?: number;
@@ -444,6 +445,19 @@ const COMMAND_CASES: readonly CommandCase[] = [
expectedExitCodes: [0, 1],
expectedNonzeroOutputIncludes: ['"ok"', '"gateway_transport_error"'],
},
{
id: "gatewayHealthJsonConnected",
name: "gateway health --json (connected)",
args: ["gateway", "health", "--json"],
presets: [],
stateScope: "case",
},
{
id: "gatewayHealthJsonFirstDevice",
name: "gateway health --json (first device)",
args: ["gateway", "health", "--json"],
presets: [],
},
{
id: "configGetGatewayPort",
name: "config get gateway.port",
@@ -649,6 +663,8 @@ function buildConfigFixture(commandCase: CommandCase): Record<string, unknown> |
if (
commandCase.id !== "configGetGatewayPort" &&
commandCase.id !== "gatewayHealthJson" &&
commandCase.id !== "gatewayHealthJsonConnected" &&
commandCase.id !== "gatewayHealthJsonFirstDevice" &&
commandCase.id !== "health" &&
commandCase.id !== "healthJson"
) {
@@ -717,8 +733,10 @@ async function runSample(params: {
cpuProfDir?: string;
heapProfDir?: string;
rssHookPath: string;
runRoot?: string;
}): Promise<Sample> {
const runRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-"));
const runRoot = params.runRoot ?? mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-"));
const ownsRunRoot = params.runRoot == null;
const stateDir = path.join(runRoot, ".openclaw");
const configPath = path.join(stateDir, "openclaw.json");
const configFixture = buildConfigFixture(params.commandCase);
@@ -849,7 +867,9 @@ async function runSample(params: {
});
});
} finally {
rmSync(runRoot, { recursive: true, force: true });
if (ownsRunRoot) {
rmSync(runRoot, { recursive: true, force: true });
}
}
}
@@ -939,14 +959,24 @@ async function runCase(params: {
}): Promise<Sample[]> {
const samples: Sample[] = [];
const totalRuns = params.warmup + params.runs;
for (let i = 0; i < totalRuns; i += 1) {
const sample = await runSample(params);
if (i < params.warmup) {
continue;
const caseRunRoot =
params.commandCase.stateScope === "case"
? mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-"))
: undefined;
try {
for (let i = 0; i < totalRuns; i += 1) {
const sample = await runSample({ ...params, runRoot: caseRunRoot });
if (i < params.warmup) {
continue;
}
samples.push(sample);
}
return samples;
} finally {
if (caseRunRoot) {
rmSync(caseRunRoot, { recursive: true, force: true });
}
samples.push(sample);
}
return samples;
}
function tailLines(value: string, maxLines: number): string {
test/scripts/bench-cli-startup.test.ts
+28 -10
View File
@@ -462,6 +462,18 @@ describe("bench-cli-startup", () => {
args: ["gateway", "health", "--json"],
presets: ["real"],
},
{
id: "gatewayHealthJsonConnected",
name: "gateway health --json (connected)",
args: ["gateway", "health", "--json"],
presets: [],
},
{
id: "gatewayHealthJsonFirstDevice",
name: "gateway health --json (first device)",
args: ["gateway", "health", "--json"],
presets: [],
},
{ id: "health", name: "health", args: ["health"], presets: ["startup", "real"] },
{
id: "healthJson",
@@ -485,16 +497,22 @@ describe("bench-cli-startup", () => {
expect(testing.parseGatewayPortEnv("::1")).toBe(32123);
expect(testing.parseGatewayPortEnv("[::1]")).toBe(32123);
expect(
withEnv({ OPENCLAW_GATEWAY_PORT: "45678" }, () =>
testing.buildConfigFixture({
id: "gatewayHealthJson",
name: "gateway health --json",
args: ["gateway", "health", "--json"],
presets: ["real"],
}),
),
).toMatchObject({ gateway: { port: 45678 } });
for (const id of [
"gatewayHealthJson",
"gatewayHealthJsonConnected",
"gatewayHealthJsonFirstDevice",
]) {
expect(
withEnv({ OPENCLAW_GATEWAY_PORT: "45678" }, () =>
testing.buildConfigFixture({
id,
name: "gateway health --json",
args: ["gateway", "health", "--json"],
presets: [],
}),
),
).toMatchObject({ gateway: { port: 45678 } });
}
for (const invalid of ["45678abc", "127.0.0.1:45678abc"]) {
expect(() =>
test/scripts/cli-startup-bench-spawner.test.ts
+103
View File
@@ -34,6 +34,109 @@ describe("CLI startup benchmark script spawners", () => {
);
});
it("reuses warmed state for gateway health while isolating first-device samples", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-state-scope-test-"));
try {
const fixturePath = path.join(tmpDir, "record-home.mjs");
const homeLogPath = path.join(tmpDir, "homes.log");
fs.writeFileSync(
fixturePath,
[
'import { appendFileSync } from "node:fs";',
"appendFileSync(process.env.OPENCLAW_BENCH_HOME_LOG, `${process.env.HOME}\\n`);",
"console.log('{\"ok\":true}');",
"",
].join("\n"),
);
const runCase = (caseId: string) => {
fs.rmSync(homeLogPath, { force: true });
execFileSync(
process.execPath,
[
"--import",
"tsx",
"scripts/bench-cli-startup.ts",
"--entry",
fixturePath,
"--case",
caseId,
"--runs",
"2",
"--warmup",
"1",
],
{
cwd: process.cwd(),
env: {
...process.env,
OPENCLAW_BENCH_HOME_LOG: homeLogPath,
},
stdio: "pipe",
},
);
return fs.readFileSync(homeLogPath, "utf8").trim().split("\n");
};
const warmedHomes = runCase("gatewayHealthJsonConnected");
expect(warmedHomes).toHaveLength(3);
expect(new Set(warmedHomes).size).toBe(1);
expect(warmedHomes.every((home) => !fs.existsSync(home))).toBe(true);
for (const caseId of ["gatewayHealthJson", "gatewayHealthJsonFirstDevice"]) {
const sampleHomes = runCase(caseId);
expect(sampleHomes).toHaveLength(3);
expect(new Set(sampleHomes).size).toBe(3);
expect(sampleHomes.every((home) => !fs.existsSync(home))).toBe(true);
}
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
it("requires connected gateway health probes to exit successfully", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-connected-test-"));
try {
const fixturePath = path.join(tmpDir, "transport-error.mjs");
fs.writeFileSync(
fixturePath,
[
'console.log(\'{"ok":false,"gateway_transport_error":"closed"}\');',
"process.exitCode = 1;",
"",
].join("\n"),
);
const runCase = (caseId: string) =>
spawnSync(
process.execPath,
[
"--import",
"tsx",
"scripts/bench-cli-startup.ts",
"--entry",
fixturePath,
"--case",
caseId,
"--runs",
"1",
"--warmup",
"0",
],
{ cwd: process.cwd(), encoding: "utf8" },
);
expect(runCase("gatewayHealthJson").status).toBe(0);
for (const caseId of ["gatewayHealthJsonConnected", "gatewayHealthJsonFirstDevice"]) {
const result = runCase(caseId);
expect(result.status).toBe(1);
expect(result.stderr).toContain(`${caseId} sample 1: exited with code 1`);
}
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
it("does not require unrelated fixture cases for a narrowed preset", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-test-"));
try {
test/scripts/openclaw-performance-workflow.test.ts
+7
View File
@@ -254,6 +254,13 @@ describe("OpenClaw performance workflow", () => {
expect(run.indexOf(probeCap)).toBeLessThan(run.indexOf(boundedProbe));
});
it("measures warmed and first-device gateway health separately", () => {
const run = findStep("Run OpenClaw source performance probes", "source_performance").run ?? "";
expect(run).toContain("--case gatewayHealthJsonConnected \\");
expect(run).toContain("--case gatewayHealthJsonFirstDevice \\");
});
it("isolates required publication in a fresh artifact-consuming job", () => {
const workflow = readWorkflow();
const publisher = workflow.jobs?.publish;