perf(update): reuse dev preflight build cache (#117246)

This commit is contained in:
Peter Steinberger
2026-07-31 22:59:44 -07:00
committed by GitHub
parent 1f40646f7e
commit 137050cedd
7 changed files with 120 additions and 27 deletions
+3 -3
View File
@@ -251,7 +251,7 @@ returns the latest sentinel.
Dev only.
</Step>
<Step title="Preflight build (dev only)">
Runs the TypeScript build in a temp worktree. If the tip fails, walks back up to 10 commits to find the newest buildable commit. Set `OPENCLAW_UPDATE_PREFLIGHT_LINT=1` to also run lint during this preflight; lint runs in constrained serial mode because user update hosts are often smaller than CI runners.
Runs the TypeScript build in a temp worktree. If the tip fails, walks back up to 10 commits to find the newest buildable commit. Content-addressed declaration outputs from the successful candidate are reused by the final checkout build; rebased source changes automatically invalidate the affected cache groups. Set `OPENCLAW_UPDATE_PREFLIGHT_LINT=1` to also run lint during this preflight; lint runs in constrained serial mode because user update hosts are often smaller than CI runners.
</Step>
<Step title="Rebase">
Rebases onto the selected commit (dev only).
@@ -259,8 +259,8 @@ returns the latest sentinel.
<Step title="Install dependencies">
Uses the repo package manager. For pnpm checkouts, the updater bootstraps `pnpm` on demand (via `corepack` first, then a temporary `npm install pnpm@11` fallback) instead of running `npm run build` inside a pnpm workspace. If pnpm bootstrap still fails, the updater stops early with a package-manager-specific error instead of trying `npm run build` in the checkout.
</Step>
<Step title="Build Control UI">
Builds the gateway and the Control UI.
<Step title="Build checkout">
Builds the gateway and Control UI once in the final checkout. The updater runs the standalone Control UI build only when a target build omitted those assets or doctor later removes them.
</Step>
<Step title="Run doctor">
`openclaw doctor` runs as the final safe-update check.
+15 -3
View File
@@ -635,9 +635,21 @@ function normalizePortablePath(filePath) {
return filePath.replaceAll("\\", "/");
}
function resolveCachePaths(rootDir, step) {
function resolveBuildCacheRoot(rootDir, env) {
// Dev update preflight and final builds run in separate worktrees. A shared
// root lets content signatures decide reuse without relocating built trees.
const configuredRoot = env?.BUILD_ALL_CACHE_ROOT?.trim();
if (!configuredRoot) {
return path.resolve(rootDir, ".artifacts/build-all-cache");
}
return path.isAbsolute(configuredRoot)
? path.normalize(configuredRoot)
: path.resolve(rootDir, configuredRoot);
}
function resolveCachePaths(rootDir, step, env) {
const safeLabel = step.label.replace(/[^a-zA-Z0-9._-]+/g, "_");
const cacheDir = path.resolve(rootDir, ".artifacts/build-all-cache", safeLabel);
const cacheDir = path.join(resolveBuildCacheRoot(rootDir, env), safeLabel);
return {
cacheDir,
outputRoot: path.join(cacheDir, "outputs"),
@@ -703,7 +715,7 @@ export function resolveBuildAllStepCacheState(step, params = {}) {
step.cache.env ?? [],
params.env ?? process.env,
);
const { outputRoot, stampPath } = resolveCachePaths(rootDir, step);
const { outputRoot, stampPath } = resolveCachePaths(rootDir, step, params.env ?? process.env);
const stamp = readCacheStamp(stampPath, fsImpl);
const outputFiles = listCacheFiles(rootDir, step.cache.outputs, fsImpl);
const relativeOutputFiles = outputFiles.map((file) => portableRelativePath(rootDir, file));
+10 -3
View File
@@ -42,13 +42,20 @@ function resolveBuildNodeOptions(baseOptions: string | undefined): string {
return current.replace(/(?:^|\s)--max-old-space-size=\d+(?=\s|$)/, ` ${desired}`).trim();
}
export function resolveBuildEnv(env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv | undefined {
export function resolveBuildEnv(
env?: NodeJS.ProcessEnv,
buildCacheRoot?: string,
): NodeJS.ProcessEnv | undefined {
const currentNodeOptions = env?.NODE_OPTIONS ?? process.env.NODE_OPTIONS;
const nextNodeOptions = resolveBuildNodeOptions(currentNodeOptions);
if (nextNodeOptions === currentNodeOptions) {
if (nextNodeOptions === currentNodeOptions && !buildCacheRoot) {
return env;
}
return { ...env, NODE_OPTIONS: nextNodeOptions };
return {
...env,
NODE_OPTIONS: nextNodeOptions,
...(buildCacheRoot ? { BUILD_ALL_CACHE_ROOT: buildCacheRoot } : {}),
};
}
export function resolveInstallEnv(
+1 -1
View File
@@ -392,7 +392,7 @@ async function testPreflightCandidates(params: {
`preflight build (${shortSha})`,
managerScriptArgs(manager.manager, "build"),
params.worktreeDir,
resolveBuildEnv(manager.env),
resolveBuildEnv(manager.env, path.join(params.gitRoot, ".artifacts", "build-all-cache")),
),
);
params.steps.push(buildStep);
+19 -8
View File
@@ -70,7 +70,7 @@ export async function runGitUpdate(params: {
const branch = await readBranchName(runCommand, gitRoot, timeoutMs);
const hasDevTargetRef = channel === "dev" && Boolean(opts.devTargetRef?.trim());
const needsCheckoutMain = channel === "dev" && !hasDevTargetRef && branch !== DEV_BRANCH;
const totalSteps = channel === "dev" ? (needsCheckoutMain ? 12 : 11) : 10;
const totalSteps = channel === "dev" ? (needsCheckoutMain ? 11 : 10) : 9;
const steps: UpdateStepResult[] = [];
let stepIndex = 0;
const step = (
@@ -389,7 +389,10 @@ export async function runGitUpdate(params: {
"build",
managerScriptArgs(manager.manager, "build"),
gitRoot,
resolveBuildEnv(manager.env),
resolveBuildEnv(
manager.env,
channel === "dev" ? path.join(gitRoot, ".artifacts", "build-all-cache") : undefined,
),
),
);
steps.push(buildStep);
@@ -410,12 +413,20 @@ export async function runGitUpdate(params: {
if (buildCleanCheck.stdoutTail?.trim()) {
return await rollbackError("build-dirty");
}
const uiBuildStep = await runStep(
step("ui:build", managerScriptArgs(manager.manager, "ui:build"), gitRoot, manager.env),
);
steps.push(uiBuildStep);
if (uiBuildStep.exitCode !== 0) {
return await rollbackError("ui-build-failed");
const builtUiIndexHealth = await resolveControlUiDistIndexHealth({ root: gitRoot });
if (!builtUiIndexHealth.exists) {
const uiBuildStep = await runStep(
step(
"ui:build (build fallback)",
managerScriptArgs(manager.manager, "ui:build"),
gitRoot,
manager.env,
),
);
steps.push(uiBuildStep);
if (uiBuildStep.exitCode !== 0) {
return await rollbackError("ui-build-failed");
}
}
const doctorEntry = path.join(gitRoot, "openclaw.mjs");
+29 -9
View File
@@ -1849,7 +1849,7 @@ describe("runGatewayUpdate", () => {
expect(calls).toContain("pnpm install");
expect(calls).toContain("pnpm build");
expect(calls).not.toContain("pnpm lint");
expect(calls).toContain("pnpm ui:build");
expect(calls).not.toContain("pnpm ui:build");
expect(pnpmEnvPaths.filter((envPath) => envPath.includes("openclaw-update-pnpm-"))).not.toEqual(
[],
);
@@ -2262,10 +2262,11 @@ describe("runGatewayUpdate", () => {
expect(cleanupStep?.stderrTail ?? "").toContain("fallback cleanup removed preflight tree");
});
it("adds heap headroom to pnpm build steps during dev updates", async () => {
it("shares the build cache while adding heap headroom to dev builds", async () => {
await setupGitPackageManagerFixture();
const upstreamSha = "upstream123";
const buildNodeOptions: string[] = [];
const buildCacheRoots: string[] = [];
const doctorNodePath = await resolveStableNodePath(process.execPath);
const doctorCommand = `${doctorNodePath} ${path.join(tempDir, "openclaw.mjs")} doctor --non-interactive --fix`;
@@ -2327,6 +2328,7 @@ describe("runGatewayUpdate", () => {
}
if (key === "pnpm build") {
buildNodeOptions.push(options?.env?.NODE_OPTIONS ?? "");
buildCacheRoots.push(options?.env?.BUILD_ALL_CACHE_ROOT ?? "");
return { stdout: "", stderr: "", code: 0 };
}
if (
@@ -2352,6 +2354,10 @@ describe("runGatewayUpdate", () => {
expect(result.status).toBe("ok");
expect(buildNodeOptions).toHaveLength(2);
expect(buildNodeOptions).toEqual(["--max-old-space-size=8192", "--max-old-space-size=8192"]);
expect(buildCacheRoots).toEqual([
path.join(tempDir, ".artifacts", "build-all-cache"),
path.join(tempDir, ".artifacts", "build-all-cache"),
]);
});
it("pins dev updates to an explicit target ref when requested", async () => {
await setupGitPackageManagerFixture();
@@ -3314,11 +3320,31 @@ describe("runGatewayUpdate", () => {
const result = await runWithCommand(runCommand, { channel: "stable" });
expect(result.status).toBe("ok");
expect(getUiBuildCount()).toBe(2);
expect(getUiBuildCount()).toBe(1);
expect(await pathExists(uiIndexPath)).toBe(true);
expect(calls).toContain(doctorKey);
});
it("builds Control UI separately only when the checkout build omitted it", async () => {
await setupGitCheckout({ packageManager: "pnpm@8.0.0" });
const uiIndexPath = path.join(tempDir, "dist", "control-ui", "index.html");
const stableTag = "v1.0.1-1";
const { runCommand, calls, doctorKey, getUiBuildCount } = await createStableTagRunner({
stableTag,
uiIndexPath,
onUiBuild: async () => {
await fs.mkdir(path.dirname(uiIndexPath), { recursive: true });
await fs.writeFile(uiIndexPath, "<html>built</html>", "utf-8");
},
});
const result = await runWithCommand(runCommand, { channel: "stable" });
expect(result.status).toBe("ok");
expect(getUiBuildCount()).toBe(1);
expect(calls.indexOf("pnpm ui:build")).toBeLessThan(calls.indexOf(doctorKey));
});
it("fails when UI assets are still missing after post-doctor repair", async () => {
await setupGitCheckout({ packageManager: "pnpm@8.0.0" });
const uiIndexPath = await setupUiIndex();
@@ -3327,12 +3353,6 @@ describe("runGatewayUpdate", () => {
const { runCommand } = await createStableTagRunner({
stableTag,
uiIndexPath,
onUiBuild: async (count) => {
if (count === 1) {
await fs.mkdir(path.dirname(uiIndexPath), { recursive: true });
await fs.writeFile(uiIndexPath, "<html>built</html>", "utf-8");
}
},
onDoctor: removeControlUiAssets,
});
+43
View File
@@ -736,6 +736,49 @@ describe("build-all timing output", () => {
});
describe("resolveBuildAllStepCacheState", () => {
it("shares content-addressed outputs across checkout roots", () => {
const cacheRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-shared-build-cache-"));
const firstRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-source-"));
const secondRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-target-"));
const step = {
label: "cached",
cache: {
inputs: ["src"],
outputs: ["dist"],
restore: "always" as const,
},
};
const env = { BUILD_ALL_CACHE_ROOT: cacheRoot };
try {
for (const rootDir of [firstRoot, secondRoot]) {
fs.mkdirSync(path.join(rootDir, "src"), { recursive: true });
fs.writeFileSync(path.join(rootDir, "src/input.ts"), "same input");
}
fs.mkdirSync(path.join(firstRoot, "dist"), { recursive: true });
fs.writeFileSync(path.join(firstRoot, "dist/output.js"), "cached output");
const sourceState = resolveBuildAllStepCacheState(step, { rootDir: firstRoot, env });
writeBuildAllStepCacheStamp(
step,
resolveBuildAllStepCacheStampState(step, sourceState, { rootDir: firstRoot }),
{ rootDir: firstRoot },
);
const targetState = resolveBuildAllStepCacheState(step, { rootDir: secondRoot, env });
expect(targetState).toMatchObject({ fresh: true, restorable: true });
expect(targetState.outputRoot).toBe(path.join(cacheRoot, "cached", "outputs"));
expect(restoreBuildAllStepCacheOutputs(targetState, { rootDir: secondRoot })).toBe(true);
expect(fs.readFileSync(path.join(secondRoot, "dist/output.js"), "utf8")).toBe(
"cached output",
);
} finally {
fs.rmSync(cacheRoot, { force: true, recursive: true });
fs.rmSync(firstRoot, { force: true, recursive: true });
fs.rmSync(secondRoot, { force: true, recursive: true });
}
});
it("invalidates only declaration groups that depend on the changed module", () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-tsdown-group-cache-"));
const ai = getBuildAllStep("tsdown-ai");