From 137050cedd32d469833b18e80f44ef66f216bba3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 31 Jul 2026 22:59:44 -0700 Subject: [PATCH] perf(update): reuse dev preflight build cache (#117246) --- docs/cli/update.md | 6 ++-- scripts/build-all.mjs | 18 ++++++++-- src/infra/update-runner-git-commands.ts | 13 +++++-- src/infra/update-runner-git-preflight.ts | 2 +- src/infra/update-runner-git.ts | 27 ++++++++++----- src/infra/update-runner.test.ts | 38 ++++++++++++++++----- test/scripts/build-all.test.ts | 43 ++++++++++++++++++++++++ 7 files changed, 120 insertions(+), 27 deletions(-) diff --git a/docs/cli/update.md b/docs/cli/update.md index 2b7a77fd6364..ad80ba6f6cbf 100644 --- a/docs/cli/update.md +++ b/docs/cli/update.md @@ -251,7 +251,7 @@ returns the latest sentinel. Dev only. - Runs the TypeScript build in a temp worktree. If the tip fails, walks back up to 10 commits to find the newest buildable commit. Set `OPENCLAW_UPDATE_PREFLIGHT_LINT=1` to also run lint during this preflight; lint runs in constrained serial mode because user update hosts are often smaller than CI runners. + Runs the TypeScript build in a temp worktree. If the tip fails, walks back up to 10 commits to find the newest buildable commit. Content-addressed declaration outputs from the successful candidate are reused by the final checkout build; rebased source changes automatically invalidate the affected cache groups. Set `OPENCLAW_UPDATE_PREFLIGHT_LINT=1` to also run lint during this preflight; lint runs in constrained serial mode because user update hosts are often smaller than CI runners. Rebases onto the selected commit (dev only). @@ -259,8 +259,8 @@ returns the latest sentinel. Uses the repo package manager. For pnpm checkouts, the updater bootstraps `pnpm` on demand (via `corepack` first, then a temporary `npm install pnpm@11` fallback) instead of running `npm run build` inside a pnpm workspace. If pnpm bootstrap still fails, the updater stops early with a package-manager-specific error instead of trying `npm run build` in the checkout. - - Builds the gateway and the Control UI. + + Builds the gateway and Control UI once in the final checkout. The updater runs the standalone Control UI build only when a target build omitted those assets or doctor later removes them. `openclaw doctor` runs as the final safe-update check. diff --git a/scripts/build-all.mjs b/scripts/build-all.mjs index 830b7088c4ec..906aa05282c3 100644 --- a/scripts/build-all.mjs +++ b/scripts/build-all.mjs @@ -635,9 +635,21 @@ function normalizePortablePath(filePath) { return filePath.replaceAll("\\", "/"); } -function resolveCachePaths(rootDir, step) { +function resolveBuildCacheRoot(rootDir, env) { + // Dev update preflight and final builds run in separate worktrees. A shared + // root lets content signatures decide reuse without relocating built trees. + const configuredRoot = env?.BUILD_ALL_CACHE_ROOT?.trim(); + if (!configuredRoot) { + return path.resolve(rootDir, ".artifacts/build-all-cache"); + } + return path.isAbsolute(configuredRoot) + ? path.normalize(configuredRoot) + : path.resolve(rootDir, configuredRoot); +} + +function resolveCachePaths(rootDir, step, env) { const safeLabel = step.label.replace(/[^a-zA-Z0-9._-]+/g, "_"); - const cacheDir = path.resolve(rootDir, ".artifacts/build-all-cache", safeLabel); + const cacheDir = path.join(resolveBuildCacheRoot(rootDir, env), safeLabel); return { cacheDir, outputRoot: path.join(cacheDir, "outputs"), @@ -703,7 +715,7 @@ export function resolveBuildAllStepCacheState(step, params = {}) { step.cache.env ?? [], params.env ?? process.env, ); - const { outputRoot, stampPath } = resolveCachePaths(rootDir, step); + const { outputRoot, stampPath } = resolveCachePaths(rootDir, step, params.env ?? process.env); const stamp = readCacheStamp(stampPath, fsImpl); const outputFiles = listCacheFiles(rootDir, step.cache.outputs, fsImpl); const relativeOutputFiles = outputFiles.map((file) => portableRelativePath(rootDir, file)); diff --git a/src/infra/update-runner-git-commands.ts b/src/infra/update-runner-git-commands.ts index f59efff7ae28..c3cb9d3df98a 100644 --- a/src/infra/update-runner-git-commands.ts +++ b/src/infra/update-runner-git-commands.ts @@ -42,13 +42,20 @@ function resolveBuildNodeOptions(baseOptions: string | undefined): string { return current.replace(/(?:^|\s)--max-old-space-size=\d+(?=\s|$)/, ` ${desired}`).trim(); } -export function resolveBuildEnv(env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv | undefined { +export function resolveBuildEnv( + env?: NodeJS.ProcessEnv, + buildCacheRoot?: string, +): NodeJS.ProcessEnv | undefined { const currentNodeOptions = env?.NODE_OPTIONS ?? process.env.NODE_OPTIONS; const nextNodeOptions = resolveBuildNodeOptions(currentNodeOptions); - if (nextNodeOptions === currentNodeOptions) { + if (nextNodeOptions === currentNodeOptions && !buildCacheRoot) { return env; } - return { ...env, NODE_OPTIONS: nextNodeOptions }; + return { + ...env, + NODE_OPTIONS: nextNodeOptions, + ...(buildCacheRoot ? { BUILD_ALL_CACHE_ROOT: buildCacheRoot } : {}), + }; } export function resolveInstallEnv( diff --git a/src/infra/update-runner-git-preflight.ts b/src/infra/update-runner-git-preflight.ts index 73fa7123c9f5..2dda39ed6af7 100644 --- a/src/infra/update-runner-git-preflight.ts +++ b/src/infra/update-runner-git-preflight.ts @@ -392,7 +392,7 @@ async function testPreflightCandidates(params: { `preflight build (${shortSha})`, managerScriptArgs(manager.manager, "build"), params.worktreeDir, - resolveBuildEnv(manager.env), + resolveBuildEnv(manager.env, path.join(params.gitRoot, ".artifacts", "build-all-cache")), ), ); params.steps.push(buildStep); diff --git a/src/infra/update-runner-git.ts b/src/infra/update-runner-git.ts index 4348ab75ea39..f636dfef18be 100644 --- a/src/infra/update-runner-git.ts +++ b/src/infra/update-runner-git.ts @@ -70,7 +70,7 @@ export async function runGitUpdate(params: { const branch = await readBranchName(runCommand, gitRoot, timeoutMs); const hasDevTargetRef = channel === "dev" && Boolean(opts.devTargetRef?.trim()); const needsCheckoutMain = channel === "dev" && !hasDevTargetRef && branch !== DEV_BRANCH; - const totalSteps = channel === "dev" ? (needsCheckoutMain ? 12 : 11) : 10; + const totalSteps = channel === "dev" ? (needsCheckoutMain ? 11 : 10) : 9; const steps: UpdateStepResult[] = []; let stepIndex = 0; const step = ( @@ -389,7 +389,10 @@ export async function runGitUpdate(params: { "build", managerScriptArgs(manager.manager, "build"), gitRoot, - resolveBuildEnv(manager.env), + resolveBuildEnv( + manager.env, + channel === "dev" ? path.join(gitRoot, ".artifacts", "build-all-cache") : undefined, + ), ), ); steps.push(buildStep); @@ -410,12 +413,20 @@ export async function runGitUpdate(params: { if (buildCleanCheck.stdoutTail?.trim()) { return await rollbackError("build-dirty"); } - const uiBuildStep = await runStep( - step("ui:build", managerScriptArgs(manager.manager, "ui:build"), gitRoot, manager.env), - ); - steps.push(uiBuildStep); - if (uiBuildStep.exitCode !== 0) { - return await rollbackError("ui-build-failed"); + const builtUiIndexHealth = await resolveControlUiDistIndexHealth({ root: gitRoot }); + if (!builtUiIndexHealth.exists) { + const uiBuildStep = await runStep( + step( + "ui:build (build fallback)", + managerScriptArgs(manager.manager, "ui:build"), + gitRoot, + manager.env, + ), + ); + steps.push(uiBuildStep); + if (uiBuildStep.exitCode !== 0) { + return await rollbackError("ui-build-failed"); + } } const doctorEntry = path.join(gitRoot, "openclaw.mjs"); diff --git a/src/infra/update-runner.test.ts b/src/infra/update-runner.test.ts index d68bc6f1ee66..ff735c603c46 100644 --- a/src/infra/update-runner.test.ts +++ b/src/infra/update-runner.test.ts @@ -1849,7 +1849,7 @@ describe("runGatewayUpdate", () => { expect(calls).toContain("pnpm install"); expect(calls).toContain("pnpm build"); expect(calls).not.toContain("pnpm lint"); - expect(calls).toContain("pnpm ui:build"); + expect(calls).not.toContain("pnpm ui:build"); expect(pnpmEnvPaths.filter((envPath) => envPath.includes("openclaw-update-pnpm-"))).not.toEqual( [], ); @@ -2262,10 +2262,11 @@ describe("runGatewayUpdate", () => { expect(cleanupStep?.stderrTail ?? "").toContain("fallback cleanup removed preflight tree"); }); - it("adds heap headroom to pnpm build steps during dev updates", async () => { + it("shares the build cache while adding heap headroom to dev builds", async () => { await setupGitPackageManagerFixture(); const upstreamSha = "upstream123"; const buildNodeOptions: string[] = []; + const buildCacheRoots: string[] = []; const doctorNodePath = await resolveStableNodePath(process.execPath); const doctorCommand = `${doctorNodePath} ${path.join(tempDir, "openclaw.mjs")} doctor --non-interactive --fix`; @@ -2327,6 +2328,7 @@ describe("runGatewayUpdate", () => { } if (key === "pnpm build") { buildNodeOptions.push(options?.env?.NODE_OPTIONS ?? ""); + buildCacheRoots.push(options?.env?.BUILD_ALL_CACHE_ROOT ?? ""); return { stdout: "", stderr: "", code: 0 }; } if ( @@ -2352,6 +2354,10 @@ describe("runGatewayUpdate", () => { expect(result.status).toBe("ok"); expect(buildNodeOptions).toHaveLength(2); expect(buildNodeOptions).toEqual(["--max-old-space-size=8192", "--max-old-space-size=8192"]); + expect(buildCacheRoots).toEqual([ + path.join(tempDir, ".artifacts", "build-all-cache"), + path.join(tempDir, ".artifacts", "build-all-cache"), + ]); }); it("pins dev updates to an explicit target ref when requested", async () => { await setupGitPackageManagerFixture(); @@ -3314,11 +3320,31 @@ describe("runGatewayUpdate", () => { const result = await runWithCommand(runCommand, { channel: "stable" }); expect(result.status).toBe("ok"); - expect(getUiBuildCount()).toBe(2); + expect(getUiBuildCount()).toBe(1); expect(await pathExists(uiIndexPath)).toBe(true); expect(calls).toContain(doctorKey); }); + it("builds Control UI separately only when the checkout build omitted it", async () => { + await setupGitCheckout({ packageManager: "pnpm@8.0.0" }); + const uiIndexPath = path.join(tempDir, "dist", "control-ui", "index.html"); + const stableTag = "v1.0.1-1"; + const { runCommand, calls, doctorKey, getUiBuildCount } = await createStableTagRunner({ + stableTag, + uiIndexPath, + onUiBuild: async () => { + await fs.mkdir(path.dirname(uiIndexPath), { recursive: true }); + await fs.writeFile(uiIndexPath, "built", "utf-8"); + }, + }); + + const result = await runWithCommand(runCommand, { channel: "stable" }); + + expect(result.status).toBe("ok"); + expect(getUiBuildCount()).toBe(1); + expect(calls.indexOf("pnpm ui:build")).toBeLessThan(calls.indexOf(doctorKey)); + }); + it("fails when UI assets are still missing after post-doctor repair", async () => { await setupGitCheckout({ packageManager: "pnpm@8.0.0" }); const uiIndexPath = await setupUiIndex(); @@ -3327,12 +3353,6 @@ describe("runGatewayUpdate", () => { const { runCommand } = await createStableTagRunner({ stableTag, uiIndexPath, - onUiBuild: async (count) => { - if (count === 1) { - await fs.mkdir(path.dirname(uiIndexPath), { recursive: true }); - await fs.writeFile(uiIndexPath, "built", "utf-8"); - } - }, onDoctor: removeControlUiAssets, }); diff --git a/test/scripts/build-all.test.ts b/test/scripts/build-all.test.ts index 99c355f5d401..bf207ef71835 100644 --- a/test/scripts/build-all.test.ts +++ b/test/scripts/build-all.test.ts @@ -736,6 +736,49 @@ describe("build-all timing output", () => { }); describe("resolveBuildAllStepCacheState", () => { + it("shares content-addressed outputs across checkout roots", () => { + const cacheRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-shared-build-cache-")); + const firstRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-source-")); + const secondRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-target-")); + const step = { + label: "cached", + cache: { + inputs: ["src"], + outputs: ["dist"], + restore: "always" as const, + }, + }; + const env = { BUILD_ALL_CACHE_ROOT: cacheRoot }; + + try { + for (const rootDir of [firstRoot, secondRoot]) { + fs.mkdirSync(path.join(rootDir, "src"), { recursive: true }); + fs.writeFileSync(path.join(rootDir, "src/input.ts"), "same input"); + } + fs.mkdirSync(path.join(firstRoot, "dist"), { recursive: true }); + fs.writeFileSync(path.join(firstRoot, "dist/output.js"), "cached output"); + + const sourceState = resolveBuildAllStepCacheState(step, { rootDir: firstRoot, env }); + writeBuildAllStepCacheStamp( + step, + resolveBuildAllStepCacheStampState(step, sourceState, { rootDir: firstRoot }), + { rootDir: firstRoot }, + ); + + const targetState = resolveBuildAllStepCacheState(step, { rootDir: secondRoot, env }); + expect(targetState).toMatchObject({ fresh: true, restorable: true }); + expect(targetState.outputRoot).toBe(path.join(cacheRoot, "cached", "outputs")); + expect(restoreBuildAllStepCacheOutputs(targetState, { rootDir: secondRoot })).toBe(true); + expect(fs.readFileSync(path.join(secondRoot, "dist/output.js"), "utf8")).toBe( + "cached output", + ); + } finally { + fs.rmSync(cacheRoot, { force: true, recursive: true }); + fs.rmSync(firstRoot, { force: true, recursive: true }); + fs.rmSync(secondRoot, { force: true, recursive: true }); + } + }); + it("invalidates only declaration groups that depend on the changed module", () => { const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-tsdown-group-cache-")); const ai = getBuildAllStep("tsdown-ai");