From ac110917e1356833f65e1536768cbfbe7c1938d0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 13 Aug 2026 14:30:27 +0800 Subject: [PATCH] fix(ci): stream npm pack receipt to disk (#123019) --- scripts/package-openclaw-for-docker.mts | 207 +++++++++++------- .../package-openclaw-for-docker.e2e.test.ts | 174 +++++++++++---- 2 files changed, 258 insertions(+), 123 deletions(-) diff --git a/scripts/package-openclaw-for-docker.mts b/scripts/package-openclaw-for-docker.mts index 9347666825b7..0e3bfb5dc99a 100644 --- a/scripts/package-openclaw-for-docker.mts +++ b/scripts/package-openclaw-for-docker.mts @@ -1,10 +1,10 @@ #!/usr/bin/env node -// Builds the OpenClaw package artifact used by Docker E2E. -// The script owns the build/inventory/pack sequence so local scheduler, shell -// helpers, and GitHub Actions all prepare the exact same npm tarball. +// Builds the canonical OpenClaw package artifact used by Docker E2E. import { spawn } from "node:child_process"; +import { closeSync, openSync } from "node:fs"; import fs from "node:fs/promises"; import { createRequire } from "node:module"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV } from "./lib/bundled-plugin-build-entries.mjs"; @@ -32,10 +32,10 @@ const AI_RUNTIME_BACKUP_DIR = ".openclaw-ai-package-backup"; type KillChild = (signal: NodeJS.Signals) => void; type RunOptions = { captureStdout?: boolean; - deferForwardedSignalExit?: boolean; env?: NodeJS.ProcessEnv; killAfterMs?: unknown; maxCapturedStdoutBytes?: number; + stdoutFilePath?: string; timeoutMs?: unknown; }; type CommandRunnerOptions = { @@ -48,15 +48,11 @@ type CommandRunner = ( cwd: string, options: CommandRunnerOptions, ) => Promise; -type CaptureRunnerOptions = { - deferForwardedSignalExit?: boolean; - timeoutMs?: number; -}; type RunImpl = ( command: string, args: string[], cwd: string, - options: CaptureRunnerOptions, + options: RunOptions, ) => Promise; type DocsMapLifecycle = { preparePackageDocsMap: (cwd: string) => Promise; @@ -305,6 +301,13 @@ export function parseArgs(argv: string[]) { } function run(command: string, args: string[], cwd: string, options: RunOptions = {}) { + const setupError = + options.captureStdout && options.stdoutFilePath + ? new Error("captureStdout and stdoutFilePath cannot be combined") + : forwardedSignalExitCode && new ForwardedSignalExitError(forwardedSignalExitCode); + if (setupError) { + return Promise.reject(setupError); + } return new Promise((resolve, reject) => { const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(options.timeoutMs); const resolvedKillAfterMs = resolvePackageBuildTimeoutMs( @@ -326,14 +329,22 @@ function run(command: string, args: string[], cwd: string, options: RunOptions = : process.platform === "win32" && command === "npm" ? resolveNpmRunner({ env, npmArgs: args }) : { args, command, shell: false }; - const child = spawn(invocation.command, invocation.args, { - cwd, - stdio: ["ignore", "pipe", "pipe"], - env: invocation.env ?? env, - detached: useProcessGroup, - shell: invocation.shell, - windowsVerbatimArguments: invocation.windowsVerbatimArguments, - }); + const stdoutFd = options.stdoutFilePath ? openSync(options.stdoutFilePath, "wx") : undefined; + let child: ReturnType; + try { + child = spawn(invocation.command, invocation.args, { + cwd, + stdio: ["ignore", stdoutFd ?? "pipe", "pipe"], + env: invocation.env ?? env, + detached: useProcessGroup, + shell: invocation.shell, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + } finally { + if (stdoutFd !== undefined) { + closeSync(stdoutFd); + } + } let timedOut = false; let outputLimitExceeded = false; let stdout = ""; @@ -354,10 +365,6 @@ function run(command: string, args: string[], cwd: string, options: RunOptions = } ACTIVE_CHILD_KILLERS.delete(killChild); if (forwardedSignalExitCode !== undefined && ACTIVE_CHILD_KILLERS.size === 0) { - if (options.deferForwardedSignalExit) { - reject(new ForwardedSignalExitError(forwardedSignalExitCode)); - return; - } process.exit(forwardedSignalExitCode); } if (error) { @@ -423,7 +430,7 @@ function run(command: string, args: string[], cwd: string, options: RunOptions = finish(error, value); }; if (options.captureStdout) { - child.stdout.on("data", (chunk) => { + child.stdout?.on("data", (chunk) => { if (outputLimitExceeded) { return; } @@ -437,10 +444,10 @@ function run(command: string, args: string[], cwd: string, options: RunOptions = stdout += chunkText; stdoutBytes += chunkBytes; }); - } else { - child.stdout.pipe(process.stderr, { end: false }); + } else if (!options.stdoutFilePath) { + child.stdout?.pipe(process.stderr, { end: false }); } - child.stderr.pipe(process.stderr, { end: false }); + child.stderr?.pipe(process.stderr, { end: false }); child.on("error", (error) => finish(error)); child.on("close", (status, signal) => { if (timedOut) { @@ -503,12 +510,12 @@ export async function buildPackageArtifacts( } } -export const runCommandForTest = run; - async function runCapture(command: string, args: string[], cwd: string, options: RunOptions = {}) { - return await run(command, args, cwd, { ...options, captureStdout: true }); + return await run(command, args, cwd, { ...options, captureStdout: !options.stdoutFilePath }); } +export { run as runCommandForTest, runCapture as runCaptureForTest }; + async function newestOpenClawTarball(outputDir: string, packOutput: string) { let fromOutput = ""; try { @@ -740,7 +747,6 @@ export async function prepareBundledAiRuntimePackage( ], sourceDir, { - deferForwardedSignalExit: true, timeoutMs: resolveTimeoutMs( "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", DEFAULT_PACKAGE_PACK_TIMEOUT_MS, @@ -859,11 +865,9 @@ async function loadSourcePackageLifecycle( } function packagePreparationRestoreError(error: unknown, restoreError: unknown) { - return new AggregateError( - [error, restoreError], - "Package preparation failed and source artifacts could not be restored.", - { cause: error }, - ); + return new AggregateError([error, restoreError], "Package operation and cleanup both failed.", { + cause: error, + }); } export async function packOpenClawPackageForDocker( @@ -921,6 +925,14 @@ export async function packOpenClawPackageForDocker( console.error("==> Packing OpenClaw package"); // This receipt is the package lifecycle lock; acquire it before touching CHANGELOG.md. await prepareDocsMap(sourcePath); + const deferSignalExit: KillChild = () => {}; + ACTIVE_CHILD_KILLERS.add(deferSignalExit); + const releaseSignalExit = () => { + ACTIVE_CHILD_KILLERS.delete(deferSignalExit); + if (forwardedSignalExitCode !== undefined) { + throw new ForwardedSignalExitError(forwardedSignalExitCode); + } + }; try { await prepareManifest(sourcePath); await prepareChangelog(sourcePath); @@ -933,66 +945,99 @@ export async function packOpenClawPackageForDocker( restoreChangelog, ); } catch (restoreError) { + releaseSignalExit(); throw packagePreparationRestoreError(error, restoreError); } + releaseSignalExit(); throw error; } let packOutput = ""; - let cleanupBundledAiRuntime = async () => {}; + let packageError: unknown; + let packReceiptDir: string | undefined; try { - await cleanPackedOpenClawTarballs(outputPath); - cleanupBundledAiRuntime = await prepareBundledAiRuntime( - sourcePath, + let cleanupBundledAiRuntime = async () => {}; + try { + await cleanPackedOpenClawTarballs(outputPath); + cleanupBundledAiRuntime = await prepareBundledAiRuntime( + sourcePath, + outputPath, + runCaptureImpl, + { + prepareManifest, + restoreManifest, + }, + ); + const packArgs = + packTool === "pnpm" + ? ["pack", "--silent", "--config.ignore-scripts=true", "--pack-destination", outputPath] + : [ + "pack", + ...(packageOptions.packJsonPath ? ["--json"] : []), + "--silent", + "--ignore-scripts", + "--pack-destination", + outputPath, + ]; + if (packTool === "npm" && packageOptions.packJsonPath) { + packReceiptDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-npm-pack-receipt-")); + } + const packReceiptPath = packReceiptDir ? path.join(packReceiptDir, "pack.json") : undefined; + packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, { + stdoutFilePath: packReceiptPath, + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", + DEFAULT_PACKAGE_PACK_TIMEOUT_MS, + ), + }); + if (packReceiptPath) { + packOutput = await fs.readFile(packReceiptPath, "utf8"); + } + } finally { + try { + await cleanupBundledAiRuntime(); + } finally { + await restorePackageSourceArtifacts( + sourcePath, + restoreDocsMap, + restoreManifest, + restoreChangelog, + ); + } + } + // Scan the emptied pnpm destination instead of trusting its absolute-path output. + let tarball = await newestOpenClawTarball( outputPath, - runCaptureImpl, - { - prepareManifest, - restoreManifest, - }, + packageOptions.pnpmPack ? "" : packOutput, ); - const packArgs = - packTool === "pnpm" - ? ["pack", "--silent", "--config.ignore-scripts=true", "--pack-destination", outputPath] - : [ - "pack", - ...(packageOptions.packJsonPath ? ["--json"] : []), - "--silent", - "--ignore-scripts", - "--pack-destination", - outputPath, - ]; - packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, { - deferForwardedSignalExit: true, - timeoutMs: resolveTimeoutMs( - "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", - DEFAULT_PACKAGE_PACK_TIMEOUT_MS, - ), - }); + if (packageOptions.outputName) { + const target = path.join(outputPath, packageOptions.outputName); + if (target !== tarball) { + await fs.rm(target, { force: true }); + await fs.rename(tarball, target); + tarball = target; + } + } + await writePackJson(packOutput, tarball, packageOptions.packJsonPath, sourcePath); + return tarball; + } catch (error) { + packageError = error; + throw error; } finally { try { - await cleanupBundledAiRuntime(); + if (packReceiptDir) { + try { + await fs.rm(packReceiptDir, { force: true, recursive: true }); + } catch (cleanupError) { + // oxlint-disable-next-line eslint/no-unsafe-finally -- Preserve primary and cleanup failures. + throw packageError + ? packagePreparationRestoreError(packageError, cleanupError) + : cleanupError; + } + } } finally { - await restorePackageSourceArtifacts( - sourcePath, - restoreDocsMap, - restoreManifest, - restoreChangelog, - ); + releaseSignalExit(); } } - // pnpm reports an absolute destination path. The directory was emptied before packing, - // so scan that controlled destination instead of accepting a path from command output. - let tarball = await newestOpenClawTarball(outputPath, packageOptions.pnpmPack ? "" : packOutput); - if (packageOptions.outputName) { - const target = path.join(outputPath, packageOptions.outputName); - if (target !== tarball) { - await fs.rm(target, { force: true }); - await fs.rename(tarball, target); - tarball = target; - } - } - await writePackJson(packOutput, tarball, packageOptions.packJsonPath, sourcePath); - return tarball; } export async function writePackageInventoryForDocker( diff --git a/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts b/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts index fb279da38110..3a4b94fd5448 100644 --- a/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts @@ -16,6 +16,7 @@ import { packOpenClawPackageForDocker, parseArgs, prepareBundledAiRuntimePackage, + runCaptureForTest, runCommandForTest, writePackageInventoryForDocker, } from "../../../../scripts/package-openclaw-for-docker.mts"; @@ -768,14 +769,8 @@ describe("package-openclaw-for-docker", () => { restoreDocsMap: async (cwd: string) => { calls.push(`restore-docs:${cwd}`); }, - runCaptureImpl: async ( - command: string, - args: string[], - cwd: string, - options: { deferForwardedSignalExit?: boolean }, - ) => { + runCaptureImpl: async (command: string, args: string[], cwd: string) => { calls.push(`${command}:${args.join(" ")}:${cwd}`); - expect(options.deferForwardedSignalExit).toBe(true); return "openclaw-2026.5.28.tgz\n"; }, }); @@ -955,61 +950,116 @@ describe("package-openclaw-for-docker", () => { }); it("normalizes npm 12 pack metadata for renamed package artifacts", async () => { + const sourceDir = tempDirs.make("openclaw-docker-pack-source-"); const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-json-")); const packJsonPath = path.join(outputDir, "pack.json"); + const npmPackOutput = JSON.stringify({ + openclaw: { + entryCount: 15_000, + filename: "openclaw-2026.5.28.tgz", + files: Array.from({ length: 15_000 }, (_, index) => ({ + mode: 0o644, + path: `dist/generated/package-entry-${String(index).padStart(5, "0")}.js`, + size: index, + })), + size: 7, + unpackedSize: 7, + version: "2026.5.28", + }, + }); + expect(Buffer.byteLength(npmPackOutput)).toBeGreaterThan(1024 * 1024); + const npmPackOutputPath = path.join(sourceDir, "npm-pack.json"); + fs.writeFileSync(npmPackOutputPath, npmPackOutput); try { - const tarball = await packOpenClawPackageForDocker("/repo", outputDir, { + const tarball = await packOpenClawPackageForDocker(sourceDir, outputDir, { ...skipDocsMapLifecycle, outputName: "openclaw-current.tgz", packJsonPath, prepareBundledAiRuntime: skipBundledAiRuntime, prepareChangelog: async () => {}, restoreChangelog: async () => {}, - runCaptureImpl: async ( - command: string, - args: string[], - _cwd: string, - options: { deferForwardedSignalExit?: boolean }, - ) => { - expect(command).toBe("npm"); - expect(args).toEqual([ - "pack", - "--json", - "--silent", - "--ignore-scripts", - "--pack-destination", - outputDir, - ]); - expect(options.deferForwardedSignalExit).toBe(true); + runCaptureImpl: async (_command, _args, cwd, options) => { fs.writeFileSync(path.join(outputDir, "openclaw-2026.5.28.tgz"), "package"); - return JSON.stringify({ - openclaw: { - entryCount: 1, - filename: "openclaw-2026.5.28.tgz", - size: 7, - unpackedSize: 7, - version: "2026.5.28", - }, - }); + return await runCaptureForTest( + process.execPath, + [ + "-e", + "process.stdout.write(require('node:fs').readFileSync(process.argv[1]))", + npmPackOutputPath, + ], + cwd, + options, + ); }, }); expect(tarball).toBe(path.join(outputDir, "openclaw-current.tgz")); - expect(JSON.parse(fs.readFileSync(packJsonPath, "utf8"))).toEqual([ - { - entryCount: 1, - filename: "openclaw-current.tgz", - size: 7, - unpackedSize: 7, - version: "2026.5.28", - }, - ]); + const packJson = JSON.parse(fs.readFileSync(packJsonPath, "utf8")) as Array<{ + entryCount: number; + filename: string; + files: unknown[]; + }>; + expect(packJson).toHaveLength(1); + expect(packJson[0]).toMatchObject({ + entryCount: 15_000, + filename: "openclaw-current.tgz", + }); + expect(packJson[0]?.files).toHaveLength(15_000); } finally { fs.rmSync(outputDir, { force: true, recursive: true }); } }); + it("cleans receipts without obscuring runner and parse failures", async () => { + const originalRm = fs.promises.rm.bind(fs.promises); + for (const failure of ["runner", "parse"] as const) { + for (const cleanupFails of [false, true]) { + const outputDir = tempDirs.make(`openclaw-docker-pack-${failure}-`); + const cleanupError = new Error("receipt cleanup failed"); + let receiptPath = ""; + const rmSpy = vi.spyOn(fs.promises, "rm").mockImplementation(async (target, options) => { + if (cleanupFails && receiptPath && target === path.dirname(receiptPath)) { + throw cleanupError; + } + return await originalRm(target, options); + }); + try { + const packPromise = packOpenClawPackageForDocker("/repo", outputDir, { + ...skipDocsMapLifecycle, + packJsonPath: path.join(outputDir, "pack.json"), + prepareBundledAiRuntime: skipBundledAiRuntime, + prepareChangelog: async () => {}, + restoreChangelog: async () => {}, + runCaptureImpl: async (_command, _args, _cwd, options) => { + receiptPath = options.stdoutFilePath ?? ""; + if (failure === "runner") throw new Error("npm pack failed"); + fs.writeFileSync(receiptPath, "not json"); + fs.writeFileSync(path.join(outputDir, "openclaw-2026.5.28.tgz"), "package"); + return ""; + }, + }); + const message = + failure === "runner" ? "npm pack failed" : "npm pack --json output was not valid JSON"; + if (cleanupFails) { + await expect(packPromise).rejects.toMatchObject({ + cause: expect.objectContaining({ message }), + errors: [expect.objectContaining({ message }), cleanupError], + message: "Package operation and cleanup both failed.", + }); + } else { + await expect(packPromise).rejects.toThrow(message); + expect(fs.existsSync(receiptPath)).toBe(false); + } + expect(receiptPath).not.toBe(""); + } finally { + rmSpy.mockRestore(); + if (receiptPath) fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true }); + } + } + } + }); + it("rejects path-like npm pack stdout before resolving Docker package tarballs", async () => { for (const filename of [ "../openclaw-2026.6.17.tgz", @@ -1297,6 +1347,46 @@ describe("package-openclaw-for-docker", () => { ).rejects.toThrow(/exceeded captured stdout limit \(1024 bytes\)/u); }); + it("writes exact stdout bytes to a file and rejects capture conflicts", async () => { + const tempDir = tempDirs.make("openclaw-package-stdout-file-"); + const stdoutFilePath = path.join(tempDir, "stdout.bin"); + const expected = Buffer.from([0, 1, 10, 13, 127, 128, 255]); + const output = await runCommandForTest( + process.execPath, + ["-e", `process.stdout.write(Buffer.from(${JSON.stringify([...expected])}))`], + process.cwd(), + { stdoutFilePath }, + ); + + expect(output).toBe(""); + expect(fs.readFileSync(stdoutFilePath)).toEqual(expected); + await expect( + runCommandForTest(process.execPath, ["-e", ""], process.cwd(), { + captureStdout: true, + stdoutFilePath: path.join(tempDir, "conflict.bin"), + }), + ).rejects.toThrow("captureStdout and stdoutFilePath cannot be combined"); + }); + + it("restores source artifacts before exiting after receipt-read termination", async () => { + if (process.platform === "win32") return; + const tempDir = tempDirs.make("openclaw-package-receipt-signal-"); + const markerPath = path.join(tempDir, "restored"); + const scriptUrl = pathToFileURL(path.resolve("scripts/package-openclaw-for-docker.mts")).href; + const runnerScript = ` +import fs from "node:fs"; +const readFile = fs.promises.readFile.bind(fs.promises); +fs.promises.readFile = async (...args) => { if (String(args[0]).endsWith("/pack.json")) { process.kill(process.pid, "SIGTERM"); await new Promise((resolve) => setTimeout(resolve, 50)); } return await readFile(...args); }; +const { packOpenClawPackageForDocker } = await import(${JSON.stringify(scriptUrl)}); +try { + await packOpenClawPackageForDocker("/repo", ${JSON.stringify(tempDir)}, { packJsonPath: "result.json", prepareBundledAiRuntime: async () => async () => {}, prepareChangelog: async () => {}, prepareDocsMap: async () => {}, prepareManifest: async () => {}, restoreChangelog: async () => {}, restoreDocsMap: async () => { fs.writeFileSync(${JSON.stringify(markerPath)}, "done"); }, restoreManifest: async () => {}, runCaptureImpl: async (_command, _args, _cwd, options) => { fs.writeFileSync(options.stdoutFilePath, '[{"filename":"openclaw-2026.5.28.tgz"}]'); fs.writeFileSync(${JSON.stringify(path.join(tempDir, "openclaw-2026.5.28.tgz"))}, "package"); return ""; } }); +} catch (error) { process.exit(error.exitCode ?? 1); } +`; + const runner = spawn(process.execPath, ["--input-type=module", "-e", runnerScript]); + expect(await waitForExit(runner, 5000)).toEqual({ signal: null, status: 143 }); + expect(fs.readFileSync(markerPath, "utf8")).toBe("done"); + }); + it("forwards external termination to active child process groups", async () => { if (process.platform === "win32") { return;