ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-23 10:55:31 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): stream npm pack receipt to disk (#123019)

Browse Source
This commit is contained in:
Vincent Koc
2026-08-13 14:30:27 +08:00
committed by GitHub
parent 5bdfa69e82
commit ac110917e1
2 changed files with 258 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/package-openclaw-for-docker.mts
+126 -81
View File
@@ -1,10 +1,10 @@
#!/usr/bin/env node
// Builds the OpenClaw package artifact used by Docker E2E.
// The script owns the build/inventory/pack sequence so local scheduler, shell
// helpers, and GitHub Actions all prepare the exact same npm tarball.
// Builds the canonical OpenClaw package artifact used by Docker E2E.
import { spawn } from "node:child_process";
import { closeSync, openSync } from "node:fs";
import fs from "node:fs/promises";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV } from "./lib/bundled-plugin-build-entries.mjs";
@@ -32,10 +32,10 @@ const AI_RUNTIME_BACKUP_DIR = ".openclaw-ai-package-backup";
type KillChild = (signal: NodeJS.Signals) => void;
type RunOptions = {
captureStdout?: boolean;
deferForwardedSignalExit?: boolean;
env?: NodeJS.ProcessEnv;
killAfterMs?: unknown;
maxCapturedStdoutBytes?: number;
stdoutFilePath?: string;
timeoutMs?: unknown;
};
type CommandRunnerOptions = {
@@ -48,15 +48,11 @@ type CommandRunner = (
cwd: string,
options: CommandRunnerOptions,
) => Promise<unknown>;
type CaptureRunnerOptions = {
deferForwardedSignalExit?: boolean;
timeoutMs?: number;
};
type RunImpl = (
command: string,
args: string[],
cwd: string,
options: CaptureRunnerOptions,
options: RunOptions,
) => Promise<string>;
type DocsMapLifecycle = {
preparePackageDocsMap: (cwd: string) => Promise<unknown>;
@@ -305,6 +301,13 @@ export function parseArgs(argv: string[]) {
}
function run(command: string, args: string[], cwd: string, options: RunOptions = {}) {
const setupError =
options.captureStdout && options.stdoutFilePath
? new Error("captureStdout and stdoutFilePath cannot be combined")
: forwardedSignalExitCode && new ForwardedSignalExitError(forwardedSignalExitCode);
if (setupError) {
return Promise.reject(setupError);
}
return new Promise<string>((resolve, reject) => {
const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(options.timeoutMs);
const resolvedKillAfterMs = resolvePackageBuildTimeoutMs(
@@ -326,14 +329,22 @@ function run(command: string, args: string[], cwd: string, options: RunOptions =
: process.platform === "win32" && command === "npm"
? resolveNpmRunner({ env, npmArgs: args })
: { args, command, shell: false };
const child = spawn(invocation.command, invocation.args, {
cwd,
stdio: ["ignore", "pipe", "pipe"],
env: invocation.env ?? env,
detached: useProcessGroup,
shell: invocation.shell,
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
});
const stdoutFd = options.stdoutFilePath ? openSync(options.stdoutFilePath, "wx") : undefined;
let child: ReturnType<typeof spawn>;
try {
child = spawn(invocation.command, invocation.args, {
cwd,
stdio: ["ignore", stdoutFd ?? "pipe", "pipe"],
env: invocation.env ?? env,
detached: useProcessGroup,
shell: invocation.shell,
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
});
} finally {
if (stdoutFd !== undefined) {
closeSync(stdoutFd);
}
}
let timedOut = false;
let outputLimitExceeded = false;
let stdout = "";
@@ -354,10 +365,6 @@ function run(command: string, args: string[], cwd: string, options: RunOptions =
}
ACTIVE_CHILD_KILLERS.delete(killChild);
if (forwardedSignalExitCode !== undefined && ACTIVE_CHILD_KILLERS.size === 0) {
if (options.deferForwardedSignalExit) {
reject(new ForwardedSignalExitError(forwardedSignalExitCode));
return;
}
process.exit(forwardedSignalExitCode);
}
if (error) {
@@ -423,7 +430,7 @@ function run(command: string, args: string[], cwd: string, options: RunOptions =
finish(error, value);
};
if (options.captureStdout) {
child.stdout.on("data", (chunk) => {
child.stdout?.on("data", (chunk) => {
if (outputLimitExceeded) {
return;
}
@@ -437,10 +444,10 @@ function run(command: string, args: string[], cwd: string, options: RunOptions =
stdout += chunkText;
stdoutBytes += chunkBytes;
});
} else {
child.stdout.pipe(process.stderr, { end: false });
} else if (!options.stdoutFilePath) {
child.stdout?.pipe(process.stderr, { end: false });
}
child.stderr.pipe(process.stderr, { end: false });
child.stderr?.pipe(process.stderr, { end: false });
child.on("error", (error) => finish(error));
child.on("close", (status, signal) => {
if (timedOut) {
@@ -503,12 +510,12 @@ export async function buildPackageArtifacts(
}
}
export const runCommandForTest = run;
async function runCapture(command: string, args: string[], cwd: string, options: RunOptions = {}) {
return await run(command, args, cwd, { ...options, captureStdout: true });
return await run(command, args, cwd, { ...options, captureStdout: !options.stdoutFilePath });
}
export { run as runCommandForTest, runCapture as runCaptureForTest };
async function newestOpenClawTarball(outputDir: string, packOutput: string) {
let fromOutput = "";
try {
@@ -740,7 +747,6 @@ export async function prepareBundledAiRuntimePackage(
],
sourceDir,
{
deferForwardedSignalExit: true,
timeoutMs: resolveTimeoutMs(
"OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS",
DEFAULT_PACKAGE_PACK_TIMEOUT_MS,
@@ -859,11 +865,9 @@ async function loadSourcePackageLifecycle(
}
function packagePreparationRestoreError(error: unknown, restoreError: unknown) {
return new AggregateError(
[error, restoreError],
"Package preparation failed and source artifacts could not be restored.",
{ cause: error },
);
return new AggregateError([error, restoreError], "Package operation and cleanup both failed.", {
cause: error,
});
}
export async function packOpenClawPackageForDocker(
@@ -921,6 +925,14 @@ export async function packOpenClawPackageForDocker(
console.error("==> Packing OpenClaw package");
// This receipt is the package lifecycle lock; acquire it before touching CHANGELOG.md.
await prepareDocsMap(sourcePath);
const deferSignalExit: KillChild = () => {};
ACTIVE_CHILD_KILLERS.add(deferSignalExit);
const releaseSignalExit = () => {
ACTIVE_CHILD_KILLERS.delete(deferSignalExit);
if (forwardedSignalExitCode !== undefined) {
throw new ForwardedSignalExitError(forwardedSignalExitCode);
}
};
try {
await prepareManifest(sourcePath);
await prepareChangelog(sourcePath);
@@ -933,66 +945,99 @@ export async function packOpenClawPackageForDocker(
restoreChangelog,
);
} catch (restoreError) {
releaseSignalExit();
throw packagePreparationRestoreError(error, restoreError);
}
releaseSignalExit();
throw error;
}
let packOutput = "";
let cleanupBundledAiRuntime = async () => {};
let packageError: unknown;
let packReceiptDir: string | undefined;
try {
await cleanPackedOpenClawTarballs(outputPath);
cleanupBundledAiRuntime = await prepareBundledAiRuntime(
sourcePath,
let cleanupBundledAiRuntime = async () => {};
try {
await cleanPackedOpenClawTarballs(outputPath);
cleanupBundledAiRuntime = await prepareBundledAiRuntime(
sourcePath,
outputPath,
runCaptureImpl,
{
prepareManifest,
restoreManifest,
},
);
const packArgs =
packTool === "pnpm"
? ["pack", "--silent", "--config.ignore-scripts=true", "--pack-destination", outputPath]
: [
"pack",
...(packageOptions.packJsonPath ? ["--json"] : []),
"--silent",
"--ignore-scripts",
"--pack-destination",
outputPath,
];
if (packTool === "npm" && packageOptions.packJsonPath) {
packReceiptDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-npm-pack-receipt-"));
}
const packReceiptPath = packReceiptDir ? path.join(packReceiptDir, "pack.json") : undefined;
packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, {
stdoutFilePath: packReceiptPath,
timeoutMs: resolveTimeoutMs(
"OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS",
DEFAULT_PACKAGE_PACK_TIMEOUT_MS,
),
});
if (packReceiptPath) {
packOutput = await fs.readFile(packReceiptPath, "utf8");
}
} finally {
try {
await cleanupBundledAiRuntime();
} finally {
await restorePackageSourceArtifacts(
sourcePath,
restoreDocsMap,
restoreManifest,
restoreChangelog,
);
}
}
// Scan the emptied pnpm destination instead of trusting its absolute-path output.
let tarball = await newestOpenClawTarball(
outputPath,
runCaptureImpl,
{
prepareManifest,
restoreManifest,
},
packageOptions.pnpmPack ? "" : packOutput,
);
const packArgs =
packTool === "pnpm"
? ["pack", "--silent", "--config.ignore-scripts=true", "--pack-destination", outputPath]
: [
"pack",
...(packageOptions.packJsonPath ? ["--json"] : []),
"--silent",
"--ignore-scripts",
"--pack-destination",
outputPath,
];
packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, {
deferForwardedSignalExit: true,
timeoutMs: resolveTimeoutMs(
"OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS",
DEFAULT_PACKAGE_PACK_TIMEOUT_MS,
),
});
if (packageOptions.outputName) {
const target = path.join(outputPath, packageOptions.outputName);
if (target !== tarball) {
await fs.rm(target, { force: true });
await fs.rename(tarball, target);
tarball = target;
}
}
await writePackJson(packOutput, tarball, packageOptions.packJsonPath, sourcePath);
return tarball;
} catch (error) {
packageError = error;
throw error;
} finally {
try {
await cleanupBundledAiRuntime();
if (packReceiptDir) {
try {
await fs.rm(packReceiptDir, { force: true, recursive: true });
} catch (cleanupError) {
// oxlint-disable-next-line eslint/no-unsafe-finally -- Preserve primary and cleanup failures.
throw packageError
? packagePreparationRestoreError(packageError, cleanupError)
: cleanupError;
}
}
} finally {
await restorePackageSourceArtifacts(
sourcePath,
restoreDocsMap,
restoreManifest,
restoreChangelog,
);
releaseSignalExit();
}
}
// pnpm reports an absolute destination path. The directory was emptied before packing,
// so scan that controlled destination instead of accepting a path from command output.
let tarball = await newestOpenClawTarball(outputPath, packageOptions.pnpmPack ? "" : packOutput);
if (packageOptions.outputName) {
const target = path.join(outputPath, packageOptions.outputName);
if (target !== tarball) {
await fs.rm(target, { force: true });
await fs.rename(tarball, target);
tarball = target;
}
}
await writePackJson(packOutput, tarball, packageOptions.packJsonPath, sourcePath);
return tarball;
}
export async function writePackageInventoryForDocker(
test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts
+132 -42
View File
@@ -16,6 +16,7 @@ import {
packOpenClawPackageForDocker,
parseArgs,
prepareBundledAiRuntimePackage,
runCaptureForTest,
runCommandForTest,
writePackageInventoryForDocker,
} from "../../../../scripts/package-openclaw-for-docker.mts";
@@ -768,14 +769,8 @@ describe("package-openclaw-for-docker", () => {
restoreDocsMap: async (cwd: string) => {
calls.push(`restore-docs:${cwd}`);
},
runCaptureImpl: async (
command: string,
args: string[],
cwd: string,
options: { deferForwardedSignalExit?: boolean },
) => {
runCaptureImpl: async (command: string, args: string[], cwd: string) => {
calls.push(`${command}:${args.join(" ")}:${cwd}`);
expect(options.deferForwardedSignalExit).toBe(true);
return "openclaw-2026.5.28.tgz\n";
},
});
@@ -955,61 +950,116 @@ describe("package-openclaw-for-docker", () => {
});
it("normalizes npm 12 pack metadata for renamed package artifacts", async () => {
const sourceDir = tempDirs.make("openclaw-docker-pack-source-");
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-json-"));
const packJsonPath = path.join(outputDir, "pack.json");
const npmPackOutput = JSON.stringify({
openclaw: {
entryCount: 15_000,
filename: "openclaw-2026.5.28.tgz",
files: Array.from({ length: 15_000 }, (_, index) => ({
mode: 0o644,
path: `dist/generated/package-entry-${String(index).padStart(5, "0")}.js`,
size: index,
})),
size: 7,
unpackedSize: 7,
version: "2026.5.28",
},
});
expect(Buffer.byteLength(npmPackOutput)).toBeGreaterThan(1024 * 1024);
const npmPackOutputPath = path.join(sourceDir, "npm-pack.json");
fs.writeFileSync(npmPackOutputPath, npmPackOutput);
try {
const tarball = await packOpenClawPackageForDocker("/repo", outputDir, {
const tarball = await packOpenClawPackageForDocker(sourceDir, outputDir, {
...skipDocsMapLifecycle,
outputName: "openclaw-current.tgz",
packJsonPath,
prepareBundledAiRuntime: skipBundledAiRuntime,
prepareChangelog: async () => {},
restoreChangelog: async () => {},
runCaptureImpl: async (
command: string,
args: string[],
_cwd: string,
options: { deferForwardedSignalExit?: boolean },
) => {
expect(command).toBe("npm");
expect(args).toEqual([
"pack",
"--json",
"--silent",
"--ignore-scripts",
"--pack-destination",
outputDir,
]);
expect(options.deferForwardedSignalExit).toBe(true);
runCaptureImpl: async (_command, _args, cwd, options) => {
fs.writeFileSync(path.join(outputDir, "openclaw-2026.5.28.tgz"), "package");
return JSON.stringify({
openclaw: {
entryCount: 1,
filename: "openclaw-2026.5.28.tgz",
size: 7,
unpackedSize: 7,
version: "2026.5.28",
},
});
return await runCaptureForTest(
process.execPath,
[
"-e",
"process.stdout.write(require('node:fs').readFileSync(process.argv[1]))",
npmPackOutputPath,
],
cwd,
options,
);
},
});
expect(tarball).toBe(path.join(outputDir, "openclaw-current.tgz"));
expect(JSON.parse(fs.readFileSync(packJsonPath, "utf8"))).toEqual([
{
entryCount: 1,
filename: "openclaw-current.tgz",
size: 7,
unpackedSize: 7,
version: "2026.5.28",
},
]);
const packJson = JSON.parse(fs.readFileSync(packJsonPath, "utf8")) as Array<{
entryCount: number;
filename: string;
files: unknown[];
}>;
expect(packJson).toHaveLength(1);
expect(packJson[0]).toMatchObject({
entryCount: 15_000,
filename: "openclaw-current.tgz",
});
expect(packJson[0]?.files).toHaveLength(15_000);
} finally {
fs.rmSync(outputDir, { force: true, recursive: true });
}
});
it("cleans receipts without obscuring runner and parse failures", async () => {
const originalRm = fs.promises.rm.bind(fs.promises);
for (const failure of ["runner", "parse"] as const) {
for (const cleanupFails of [false, true]) {
const outputDir = tempDirs.make(`openclaw-docker-pack-${failure}-`);
const cleanupError = new Error("receipt cleanup failed");
let receiptPath = "";
const rmSpy = vi.spyOn(fs.promises, "rm").mockImplementation(async (target, options) => {
if (cleanupFails && receiptPath && target === path.dirname(receiptPath)) {
throw cleanupError;
}
return await originalRm(target, options);
});
try {
const packPromise = packOpenClawPackageForDocker("/repo", outputDir, {
...skipDocsMapLifecycle,
packJsonPath: path.join(outputDir, "pack.json"),
prepareBundledAiRuntime: skipBundledAiRuntime,
prepareChangelog: async () => {},
restoreChangelog: async () => {},
runCaptureImpl: async (_command, _args, _cwd, options) => {
receiptPath = options.stdoutFilePath ?? "";
if (failure === "runner") throw new Error("npm pack failed");
fs.writeFileSync(receiptPath, "not json");
fs.writeFileSync(path.join(outputDir, "openclaw-2026.5.28.tgz"), "package");
return "";
},
});
const message =
failure === "runner" ? "npm pack failed" : "npm pack --json output was not valid JSON";
if (cleanupFails) {
await expect(packPromise).rejects.toMatchObject({
cause: expect.objectContaining({ message }),
errors: [expect.objectContaining({ message }), cleanupError],
message: "Package operation and cleanup both failed.",
});
} else {
await expect(packPromise).rejects.toThrow(message);
expect(fs.existsSync(receiptPath)).toBe(false);
}
expect(receiptPath).not.toBe("");
} finally {
rmSpy.mockRestore();
if (receiptPath) fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true });
}
}
}
});
it("rejects path-like npm pack stdout before resolving Docker package tarballs", async () => {
for (const filename of [
"../openclaw-2026.6.17.tgz",
@@ -1297,6 +1347,46 @@ describe("package-openclaw-for-docker", () => {
).rejects.toThrow(/exceeded captured stdout limit \(1024 bytes\)/u);
});
it("writes exact stdout bytes to a file and rejects capture conflicts", async () => {
const tempDir = tempDirs.make("openclaw-package-stdout-file-");
const stdoutFilePath = path.join(tempDir, "stdout.bin");
const expected = Buffer.from([0, 1, 10, 13, 127, 128, 255]);
const output = await runCommandForTest(
process.execPath,
["-e", `process.stdout.write(Buffer.from(${JSON.stringify([...expected])}))`],
process.cwd(),
{ stdoutFilePath },
);
expect(output).toBe("");
expect(fs.readFileSync(stdoutFilePath)).toEqual(expected);
await expect(
runCommandForTest(process.execPath, ["-e", ""], process.cwd(), {
captureStdout: true,
stdoutFilePath: path.join(tempDir, "conflict.bin"),
}),
).rejects.toThrow("captureStdout and stdoutFilePath cannot be combined");
});
it("restores source artifacts before exiting after receipt-read termination", async () => {
if (process.platform === "win32") return;
const tempDir = tempDirs.make("openclaw-package-receipt-signal-");
const markerPath = path.join(tempDir, "restored");
const scriptUrl = pathToFileURL(path.resolve("scripts/package-openclaw-for-docker.mts")).href;
const runnerScript = `
import fs from "node:fs";
const readFile = fs.promises.readFile.bind(fs.promises);
fs.promises.readFile = async (...args) => { if (String(args[0]).endsWith("/pack.json")) { process.kill(process.pid, "SIGTERM"); await new Promise((resolve) => setTimeout(resolve, 50)); } return await readFile(...args); };
const { packOpenClawPackageForDocker } = await import(${JSON.stringify(scriptUrl)});
try {
await packOpenClawPackageForDocker("/repo", ${JSON.stringify(tempDir)}, { packJsonPath: "result.json", prepareBundledAiRuntime: async () => async () => {}, prepareChangelog: async () => {}, prepareDocsMap: async () => {}, prepareManifest: async () => {}, restoreChangelog: async () => {}, restoreDocsMap: async () => { fs.writeFileSync(${JSON.stringify(markerPath)}, "done"); }, restoreManifest: async () => {}, runCaptureImpl: async (_command, _args, _cwd, options) => { fs.writeFileSync(options.stdoutFilePath, '[{"filename":"openclaw-2026.5.28.tgz"}]'); fs.writeFileSync(${JSON.stringify(path.join(tempDir, "openclaw-2026.5.28.tgz"))}, "package"); return ""; } });
} catch (error) { process.exit(error.exitCode ?? 1); }
`;
const runner = spawn(process.execPath, ["--input-type=module", "-e", runnerScript]);
expect(await waitForExit(runner, 5000)).toEqual({ signal: null, status: 143 });
expect(fs.readFileSync(markerPath, "utf8")).toBe("done");
});
it("forwards external termination to active child process groups", async () => {
if (process.platform === "win32") {
return;