mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
perf(update): reuse dev preflight build cache (#117246)
This commit is contained in:
committed by
GitHub
parent
1f40646f7e
commit
137050cedd
+3
-3
@@ -251,7 +251,7 @@ returns the latest sentinel.
|
||||
Dev only.
|
||||
</Step>
|
||||
<Step title="Preflight build (dev only)">
|
||||
Runs the TypeScript build in a temp worktree. If the tip fails, walks back up to 10 commits to find the newest buildable commit. Set `OPENCLAW_UPDATE_PREFLIGHT_LINT=1` to also run lint during this preflight; lint runs in constrained serial mode because user update hosts are often smaller than CI runners.
|
||||
Runs the TypeScript build in a temp worktree. If the tip fails, walks back up to 10 commits to find the newest buildable commit. Content-addressed declaration outputs from the successful candidate are reused by the final checkout build; rebased source changes automatically invalidate the affected cache groups. Set `OPENCLAW_UPDATE_PREFLIGHT_LINT=1` to also run lint during this preflight; lint runs in constrained serial mode because user update hosts are often smaller than CI runners.
|
||||
</Step>
|
||||
<Step title="Rebase">
|
||||
Rebases onto the selected commit (dev only).
|
||||
@@ -259,8 +259,8 @@ returns the latest sentinel.
|
||||
<Step title="Install dependencies">
|
||||
Uses the repo package manager. For pnpm checkouts, the updater bootstraps `pnpm` on demand (via `corepack` first, then a temporary `npm install pnpm@11` fallback) instead of running `npm run build` inside a pnpm workspace. If pnpm bootstrap still fails, the updater stops early with a package-manager-specific error instead of trying `npm run build` in the checkout.
|
||||
</Step>
|
||||
<Step title="Build Control UI">
|
||||
Builds the gateway and the Control UI.
|
||||
<Step title="Build checkout">
|
||||
Builds the gateway and Control UI once in the final checkout. The updater runs the standalone Control UI build only when a target build omitted those assets or doctor later removes them.
|
||||
</Step>
|
||||
<Step title="Run doctor">
|
||||
`openclaw doctor` runs as the final safe-update check.
|
||||
|
||||
+15
-3
@@ -635,9 +635,21 @@ function normalizePortablePath(filePath) {
|
||||
return filePath.replaceAll("\\", "/");
|
||||
}
|
||||
|
||||
function resolveCachePaths(rootDir, step) {
|
||||
function resolveBuildCacheRoot(rootDir, env) {
|
||||
// Dev update preflight and final builds run in separate worktrees. A shared
|
||||
// root lets content signatures decide reuse without relocating built trees.
|
||||
const configuredRoot = env?.BUILD_ALL_CACHE_ROOT?.trim();
|
||||
if (!configuredRoot) {
|
||||
return path.resolve(rootDir, ".artifacts/build-all-cache");
|
||||
}
|
||||
return path.isAbsolute(configuredRoot)
|
||||
? path.normalize(configuredRoot)
|
||||
: path.resolve(rootDir, configuredRoot);
|
||||
}
|
||||
|
||||
function resolveCachePaths(rootDir, step, env) {
|
||||
const safeLabel = step.label.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
||||
const cacheDir = path.resolve(rootDir, ".artifacts/build-all-cache", safeLabel);
|
||||
const cacheDir = path.join(resolveBuildCacheRoot(rootDir, env), safeLabel);
|
||||
return {
|
||||
cacheDir,
|
||||
outputRoot: path.join(cacheDir, "outputs"),
|
||||
@@ -703,7 +715,7 @@ export function resolveBuildAllStepCacheState(step, params = {}) {
|
||||
step.cache.env ?? [],
|
||||
params.env ?? process.env,
|
||||
);
|
||||
const { outputRoot, stampPath } = resolveCachePaths(rootDir, step);
|
||||
const { outputRoot, stampPath } = resolveCachePaths(rootDir, step, params.env ?? process.env);
|
||||
const stamp = readCacheStamp(stampPath, fsImpl);
|
||||
const outputFiles = listCacheFiles(rootDir, step.cache.outputs, fsImpl);
|
||||
const relativeOutputFiles = outputFiles.map((file) => portableRelativePath(rootDir, file));
|
||||
|
||||
@@ -42,13 +42,20 @@ function resolveBuildNodeOptions(baseOptions: string | undefined): string {
|
||||
return current.replace(/(?:^|\s)--max-old-space-size=\d+(?=\s|$)/, ` ${desired}`).trim();
|
||||
}
|
||||
|
||||
export function resolveBuildEnv(env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv | undefined {
|
||||
export function resolveBuildEnv(
|
||||
env?: NodeJS.ProcessEnv,
|
||||
buildCacheRoot?: string,
|
||||
): NodeJS.ProcessEnv | undefined {
|
||||
const currentNodeOptions = env?.NODE_OPTIONS ?? process.env.NODE_OPTIONS;
|
||||
const nextNodeOptions = resolveBuildNodeOptions(currentNodeOptions);
|
||||
if (nextNodeOptions === currentNodeOptions) {
|
||||
if (nextNodeOptions === currentNodeOptions && !buildCacheRoot) {
|
||||
return env;
|
||||
}
|
||||
return { ...env, NODE_OPTIONS: nextNodeOptions };
|
||||
return {
|
||||
...env,
|
||||
NODE_OPTIONS: nextNodeOptions,
|
||||
...(buildCacheRoot ? { BUILD_ALL_CACHE_ROOT: buildCacheRoot } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveInstallEnv(
|
||||
|
||||
@@ -392,7 +392,7 @@ async function testPreflightCandidates(params: {
|
||||
`preflight build (${shortSha})`,
|
||||
managerScriptArgs(manager.manager, "build"),
|
||||
params.worktreeDir,
|
||||
resolveBuildEnv(manager.env),
|
||||
resolveBuildEnv(manager.env, path.join(params.gitRoot, ".artifacts", "build-all-cache")),
|
||||
),
|
||||
);
|
||||
params.steps.push(buildStep);
|
||||
|
||||
@@ -70,7 +70,7 @@ export async function runGitUpdate(params: {
|
||||
const branch = await readBranchName(runCommand, gitRoot, timeoutMs);
|
||||
const hasDevTargetRef = channel === "dev" && Boolean(opts.devTargetRef?.trim());
|
||||
const needsCheckoutMain = channel === "dev" && !hasDevTargetRef && branch !== DEV_BRANCH;
|
||||
const totalSteps = channel === "dev" ? (needsCheckoutMain ? 12 : 11) : 10;
|
||||
const totalSteps = channel === "dev" ? (needsCheckoutMain ? 11 : 10) : 9;
|
||||
const steps: UpdateStepResult[] = [];
|
||||
let stepIndex = 0;
|
||||
const step = (
|
||||
@@ -389,7 +389,10 @@ export async function runGitUpdate(params: {
|
||||
"build",
|
||||
managerScriptArgs(manager.manager, "build"),
|
||||
gitRoot,
|
||||
resolveBuildEnv(manager.env),
|
||||
resolveBuildEnv(
|
||||
manager.env,
|
||||
channel === "dev" ? path.join(gitRoot, ".artifacts", "build-all-cache") : undefined,
|
||||
),
|
||||
),
|
||||
);
|
||||
steps.push(buildStep);
|
||||
@@ -410,12 +413,20 @@ export async function runGitUpdate(params: {
|
||||
if (buildCleanCheck.stdoutTail?.trim()) {
|
||||
return await rollbackError("build-dirty");
|
||||
}
|
||||
const uiBuildStep = await runStep(
|
||||
step("ui:build", managerScriptArgs(manager.manager, "ui:build"), gitRoot, manager.env),
|
||||
);
|
||||
steps.push(uiBuildStep);
|
||||
if (uiBuildStep.exitCode !== 0) {
|
||||
return await rollbackError("ui-build-failed");
|
||||
const builtUiIndexHealth = await resolveControlUiDistIndexHealth({ root: gitRoot });
|
||||
if (!builtUiIndexHealth.exists) {
|
||||
const uiBuildStep = await runStep(
|
||||
step(
|
||||
"ui:build (build fallback)",
|
||||
managerScriptArgs(manager.manager, "ui:build"),
|
||||
gitRoot,
|
||||
manager.env,
|
||||
),
|
||||
);
|
||||
steps.push(uiBuildStep);
|
||||
if (uiBuildStep.exitCode !== 0) {
|
||||
return await rollbackError("ui-build-failed");
|
||||
}
|
||||
}
|
||||
|
||||
const doctorEntry = path.join(gitRoot, "openclaw.mjs");
|
||||
|
||||
@@ -1849,7 +1849,7 @@ describe("runGatewayUpdate", () => {
|
||||
expect(calls).toContain("pnpm install");
|
||||
expect(calls).toContain("pnpm build");
|
||||
expect(calls).not.toContain("pnpm lint");
|
||||
expect(calls).toContain("pnpm ui:build");
|
||||
expect(calls).not.toContain("pnpm ui:build");
|
||||
expect(pnpmEnvPaths.filter((envPath) => envPath.includes("openclaw-update-pnpm-"))).not.toEqual(
|
||||
[],
|
||||
);
|
||||
@@ -2262,10 +2262,11 @@ describe("runGatewayUpdate", () => {
|
||||
expect(cleanupStep?.stderrTail ?? "").toContain("fallback cleanup removed preflight tree");
|
||||
});
|
||||
|
||||
it("adds heap headroom to pnpm build steps during dev updates", async () => {
|
||||
it("shares the build cache while adding heap headroom to dev builds", async () => {
|
||||
await setupGitPackageManagerFixture();
|
||||
const upstreamSha = "upstream123";
|
||||
const buildNodeOptions: string[] = [];
|
||||
const buildCacheRoots: string[] = [];
|
||||
const doctorNodePath = await resolveStableNodePath(process.execPath);
|
||||
const doctorCommand = `${doctorNodePath} ${path.join(tempDir, "openclaw.mjs")} doctor --non-interactive --fix`;
|
||||
|
||||
@@ -2327,6 +2328,7 @@ describe("runGatewayUpdate", () => {
|
||||
}
|
||||
if (key === "pnpm build") {
|
||||
buildNodeOptions.push(options?.env?.NODE_OPTIONS ?? "");
|
||||
buildCacheRoots.push(options?.env?.BUILD_ALL_CACHE_ROOT ?? "");
|
||||
return { stdout: "", stderr: "", code: 0 };
|
||||
}
|
||||
if (
|
||||
@@ -2352,6 +2354,10 @@ describe("runGatewayUpdate", () => {
|
||||
expect(result.status).toBe("ok");
|
||||
expect(buildNodeOptions).toHaveLength(2);
|
||||
expect(buildNodeOptions).toEqual(["--max-old-space-size=8192", "--max-old-space-size=8192"]);
|
||||
expect(buildCacheRoots).toEqual([
|
||||
path.join(tempDir, ".artifacts", "build-all-cache"),
|
||||
path.join(tempDir, ".artifacts", "build-all-cache"),
|
||||
]);
|
||||
});
|
||||
it("pins dev updates to an explicit target ref when requested", async () => {
|
||||
await setupGitPackageManagerFixture();
|
||||
@@ -3314,11 +3320,31 @@ describe("runGatewayUpdate", () => {
|
||||
const result = await runWithCommand(runCommand, { channel: "stable" });
|
||||
|
||||
expect(result.status).toBe("ok");
|
||||
expect(getUiBuildCount()).toBe(2);
|
||||
expect(getUiBuildCount()).toBe(1);
|
||||
expect(await pathExists(uiIndexPath)).toBe(true);
|
||||
expect(calls).toContain(doctorKey);
|
||||
});
|
||||
|
||||
it("builds Control UI separately only when the checkout build omitted it", async () => {
|
||||
await setupGitCheckout({ packageManager: "pnpm@8.0.0" });
|
||||
const uiIndexPath = path.join(tempDir, "dist", "control-ui", "index.html");
|
||||
const stableTag = "v1.0.1-1";
|
||||
const { runCommand, calls, doctorKey, getUiBuildCount } = await createStableTagRunner({
|
||||
stableTag,
|
||||
uiIndexPath,
|
||||
onUiBuild: async () => {
|
||||
await fs.mkdir(path.dirname(uiIndexPath), { recursive: true });
|
||||
await fs.writeFile(uiIndexPath, "<html>built</html>", "utf-8");
|
||||
},
|
||||
});
|
||||
|
||||
const result = await runWithCommand(runCommand, { channel: "stable" });
|
||||
|
||||
expect(result.status).toBe("ok");
|
||||
expect(getUiBuildCount()).toBe(1);
|
||||
expect(calls.indexOf("pnpm ui:build")).toBeLessThan(calls.indexOf(doctorKey));
|
||||
});
|
||||
|
||||
it("fails when UI assets are still missing after post-doctor repair", async () => {
|
||||
await setupGitCheckout({ packageManager: "pnpm@8.0.0" });
|
||||
const uiIndexPath = await setupUiIndex();
|
||||
@@ -3327,12 +3353,6 @@ describe("runGatewayUpdate", () => {
|
||||
const { runCommand } = await createStableTagRunner({
|
||||
stableTag,
|
||||
uiIndexPath,
|
||||
onUiBuild: async (count) => {
|
||||
if (count === 1) {
|
||||
await fs.mkdir(path.dirname(uiIndexPath), { recursive: true });
|
||||
await fs.writeFile(uiIndexPath, "<html>built</html>", "utf-8");
|
||||
}
|
||||
},
|
||||
onDoctor: removeControlUiAssets,
|
||||
});
|
||||
|
||||
|
||||
@@ -736,6 +736,49 @@ describe("build-all timing output", () => {
|
||||
});
|
||||
|
||||
describe("resolveBuildAllStepCacheState", () => {
|
||||
it("shares content-addressed outputs across checkout roots", () => {
|
||||
const cacheRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-shared-build-cache-"));
|
||||
const firstRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-source-"));
|
||||
const secondRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-cache-target-"));
|
||||
const step = {
|
||||
label: "cached",
|
||||
cache: {
|
||||
inputs: ["src"],
|
||||
outputs: ["dist"],
|
||||
restore: "always" as const,
|
||||
},
|
||||
};
|
||||
const env = { BUILD_ALL_CACHE_ROOT: cacheRoot };
|
||||
|
||||
try {
|
||||
for (const rootDir of [firstRoot, secondRoot]) {
|
||||
fs.mkdirSync(path.join(rootDir, "src"), { recursive: true });
|
||||
fs.writeFileSync(path.join(rootDir, "src/input.ts"), "same input");
|
||||
}
|
||||
fs.mkdirSync(path.join(firstRoot, "dist"), { recursive: true });
|
||||
fs.writeFileSync(path.join(firstRoot, "dist/output.js"), "cached output");
|
||||
|
||||
const sourceState = resolveBuildAllStepCacheState(step, { rootDir: firstRoot, env });
|
||||
writeBuildAllStepCacheStamp(
|
||||
step,
|
||||
resolveBuildAllStepCacheStampState(step, sourceState, { rootDir: firstRoot }),
|
||||
{ rootDir: firstRoot },
|
||||
);
|
||||
|
||||
const targetState = resolveBuildAllStepCacheState(step, { rootDir: secondRoot, env });
|
||||
expect(targetState).toMatchObject({ fresh: true, restorable: true });
|
||||
expect(targetState.outputRoot).toBe(path.join(cacheRoot, "cached", "outputs"));
|
||||
expect(restoreBuildAllStepCacheOutputs(targetState, { rootDir: secondRoot })).toBe(true);
|
||||
expect(fs.readFileSync(path.join(secondRoot, "dist/output.js"), "utf8")).toBe(
|
||||
"cached output",
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(cacheRoot, { force: true, recursive: true });
|
||||
fs.rmSync(firstRoot, { force: true, recursive: true });
|
||||
fs.rmSync(secondRoot, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("invalidates only declaration groups that depend on the changed module", () => {
|
||||
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-tsdown-group-cache-"));
|
||||
const ai = getBuildAllStep("tsdown-ai");
|
||||
|
||||
Reference in New Issue
Block a user