diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index 0185f411b367..66aac720f846 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -807,7 +807,8 @@ jobs: OPENCLAW_HOME="$gateway_home" OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" OPENCLAW_GATEWAY_PORT="$gateway_port" \ node --import tsx scripts/bench-cli-startup.ts \ - --case gatewayHealthJson \ + --case gatewayHealthJsonConnected \ + --case gatewayHealthJsonFirstDevice \ --case configGetGatewayPort \ --runs "$source_runs" \ --warmup 1 \ diff --git a/scripts/bench-cli-startup.ts b/scripts/bench-cli-startup.ts index affb00029acd..349849d8b149 100644 --- a/scripts/bench-cli-startup.ts +++ b/scripts/bench-cli-startup.ts @@ -12,6 +12,7 @@ type CommandCase = { name: string; args: string[]; presets: readonly string[]; + stateScope?: "case" | "sample"; expectedExitCodes?: readonly number[]; expectedNonzeroOutputIncludes?: readonly string[]; firstOutputBudgetMs?: number; @@ -444,6 +445,19 @@ const COMMAND_CASES: readonly CommandCase[] = [ expectedExitCodes: [0, 1], expectedNonzeroOutputIncludes: ['"ok"', '"gateway_transport_error"'], }, + { + id: "gatewayHealthJsonConnected", + name: "gateway health --json (connected)", + args: ["gateway", "health", "--json"], + presets: [], + stateScope: "case", + }, + { + id: "gatewayHealthJsonFirstDevice", + name: "gateway health --json (first device)", + args: ["gateway", "health", "--json"], + presets: [], + }, { id: "configGetGatewayPort", name: "config get gateway.port", @@ -649,6 +663,8 @@ function buildConfigFixture(commandCase: CommandCase): Record | if ( commandCase.id !== "configGetGatewayPort" && commandCase.id !== "gatewayHealthJson" && + commandCase.id !== "gatewayHealthJsonConnected" && + commandCase.id !== "gatewayHealthJsonFirstDevice" && commandCase.id !== "health" && commandCase.id !== "healthJson" ) { @@ -717,8 +733,10 @@ async function runSample(params: { cpuProfDir?: string; heapProfDir?: string; rssHookPath: string; + runRoot?: string; }): Promise { - const runRoot = mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-")); + const runRoot = params.runRoot ?? mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-")); + const ownsRunRoot = params.runRoot == null; const stateDir = path.join(runRoot, ".openclaw"); const configPath = path.join(stateDir, "openclaw.json"); const configFixture = buildConfigFixture(params.commandCase); @@ -849,7 +867,9 @@ async function runSample(params: { }); }); } finally { - rmSync(runRoot, { recursive: true, force: true }); + if (ownsRunRoot) { + rmSync(runRoot, { recursive: true, force: true }); + } } } @@ -939,14 +959,24 @@ async function runCase(params: { }): Promise { const samples: Sample[] = []; const totalRuns = params.warmup + params.runs; - for (let i = 0; i < totalRuns; i += 1) { - const sample = await runSample(params); - if (i < params.warmup) { - continue; + const caseRunRoot = + params.commandCase.stateScope === "case" + ? mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-")) + : undefined; + try { + for (let i = 0; i < totalRuns; i += 1) { + const sample = await runSample({ ...params, runRoot: caseRunRoot }); + if (i < params.warmup) { + continue; + } + samples.push(sample); + } + return samples; + } finally { + if (caseRunRoot) { + rmSync(caseRunRoot, { recursive: true, force: true }); } - samples.push(sample); } - return samples; } function tailLines(value: string, maxLines: number): string { diff --git a/test/scripts/bench-cli-startup.test.ts b/test/scripts/bench-cli-startup.test.ts index a4792df914de..506d76812331 100644 --- a/test/scripts/bench-cli-startup.test.ts +++ b/test/scripts/bench-cli-startup.test.ts @@ -462,6 +462,18 @@ describe("bench-cli-startup", () => { args: ["gateway", "health", "--json"], presets: ["real"], }, + { + id: "gatewayHealthJsonConnected", + name: "gateway health --json (connected)", + args: ["gateway", "health", "--json"], + presets: [], + }, + { + id: "gatewayHealthJsonFirstDevice", + name: "gateway health --json (first device)", + args: ["gateway", "health", "--json"], + presets: [], + }, { id: "health", name: "health", args: ["health"], presets: ["startup", "real"] }, { id: "healthJson", @@ -485,16 +497,22 @@ describe("bench-cli-startup", () => { expect(testing.parseGatewayPortEnv("::1")).toBe(32123); expect(testing.parseGatewayPortEnv("[::1]")).toBe(32123); - expect( - withEnv({ OPENCLAW_GATEWAY_PORT: "45678" }, () => - testing.buildConfigFixture({ - id: "gatewayHealthJson", - name: "gateway health --json", - args: ["gateway", "health", "--json"], - presets: ["real"], - }), - ), - ).toMatchObject({ gateway: { port: 45678 } }); + for (const id of [ + "gatewayHealthJson", + "gatewayHealthJsonConnected", + "gatewayHealthJsonFirstDevice", + ]) { + expect( + withEnv({ OPENCLAW_GATEWAY_PORT: "45678" }, () => + testing.buildConfigFixture({ + id, + name: "gateway health --json", + args: ["gateway", "health", "--json"], + presets: [], + }), + ), + ).toMatchObject({ gateway: { port: 45678 } }); + } for (const invalid of ["45678abc", "127.0.0.1:45678abc"]) { expect(() => diff --git a/test/scripts/cli-startup-bench-spawner.test.ts b/test/scripts/cli-startup-bench-spawner.test.ts index 14c6f0f43d9b..50a84d997b61 100644 --- a/test/scripts/cli-startup-bench-spawner.test.ts +++ b/test/scripts/cli-startup-bench-spawner.test.ts @@ -34,6 +34,109 @@ describe("CLI startup benchmark script spawners", () => { ); }); + it("reuses warmed state for gateway health while isolating first-device samples", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-state-scope-test-")); + try { + const fixturePath = path.join(tmpDir, "record-home.mjs"); + const homeLogPath = path.join(tmpDir, "homes.log"); + fs.writeFileSync( + fixturePath, + [ + 'import { appendFileSync } from "node:fs";', + "appendFileSync(process.env.OPENCLAW_BENCH_HOME_LOG, `${process.env.HOME}\\n`);", + "console.log('{\"ok\":true}');", + "", + ].join("\n"), + ); + + const runCase = (caseId: string) => { + fs.rmSync(homeLogPath, { force: true }); + execFileSync( + process.execPath, + [ + "--import", + "tsx", + "scripts/bench-cli-startup.ts", + "--entry", + fixturePath, + "--case", + caseId, + "--runs", + "2", + "--warmup", + "1", + ], + { + cwd: process.cwd(), + env: { + ...process.env, + OPENCLAW_BENCH_HOME_LOG: homeLogPath, + }, + stdio: "pipe", + }, + ); + return fs.readFileSync(homeLogPath, "utf8").trim().split("\n"); + }; + + const warmedHomes = runCase("gatewayHealthJsonConnected"); + expect(warmedHomes).toHaveLength(3); + expect(new Set(warmedHomes).size).toBe(1); + expect(warmedHomes.every((home) => !fs.existsSync(home))).toBe(true); + + for (const caseId of ["gatewayHealthJson", "gatewayHealthJsonFirstDevice"]) { + const sampleHomes = runCase(caseId); + expect(sampleHomes).toHaveLength(3); + expect(new Set(sampleHomes).size).toBe(3); + expect(sampleHomes.every((home) => !fs.existsSync(home))).toBe(true); + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("requires connected gateway health probes to exit successfully", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-connected-test-")); + try { + const fixturePath = path.join(tmpDir, "transport-error.mjs"); + fs.writeFileSync( + fixturePath, + [ + 'console.log(\'{"ok":false,"gateway_transport_error":"closed"}\');', + "process.exitCode = 1;", + "", + ].join("\n"), + ); + + const runCase = (caseId: string) => + spawnSync( + process.execPath, + [ + "--import", + "tsx", + "scripts/bench-cli-startup.ts", + "--entry", + fixturePath, + "--case", + caseId, + "--runs", + "1", + "--warmup", + "0", + ], + { cwd: process.cwd(), encoding: "utf8" }, + ); + + expect(runCase("gatewayHealthJson").status).toBe(0); + for (const caseId of ["gatewayHealthJsonConnected", "gatewayHealthJsonFirstDevice"]) { + const result = runCase(caseId); + expect(result.status).toBe(1); + expect(result.stderr).toContain(`${caseId} sample 1: exited with code 1`); + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + it("does not require unrelated fixture cases for a narrowed preset", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-test-")); try { diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index 0f1129dde2f9..522df47e675e 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -254,6 +254,13 @@ describe("OpenClaw performance workflow", () => { expect(run.indexOf(probeCap)).toBeLessThan(run.indexOf(boundedProbe)); }); + it("measures warmed and first-device gateway health separately", () => { + const run = findStep("Run OpenClaw source performance probes", "source_performance").run ?? ""; + + expect(run).toContain("--case gatewayHealthJsonConnected \\"); + expect(run).toContain("--case gatewayHealthJsonFirstDevice \\"); + }); + it("isolates required publication in a fresh artifact-consuming job", () => { const workflow = readWorkflow(); const publisher = workflow.jobs?.publish;