fix: preserve live updater failure diagnostics (#119096)

This commit is contained in:
Jason (Json)
2026-08-03 23:30:17 -06:00
committed by GitHub
parent 3be66d7716
commit 8ad2e06a9d
3 changed files with 671 additions and 38 deletions
@@ -48,6 +48,17 @@ const GATEWAY_PROCESS_START_RETRY_DELAY_MS = 250;
const GATEWAY_SUSPEND_TIMEOUT_MS = 10_000;
const GATEWAY_STARTUP_TRACE_ENV = "OPENCLAW_GATEWAY_STARTUP_TRACE";
const SYSTEM_LAUNCH_DAEMON_DIR = "/Library/LaunchDaemons";
const MAX_FAILURE_DIAGNOSTIC_DEPTH = 4;
const MAX_FAILURE_DIAGNOSTIC_MEMBERS = 8;
const SAFE_INVARIANT_DETAIL_KEYS = [
"exitTimeoutSeconds",
"listenerClosed",
"processExited",
"serviceBootedOut",
];
// CLI diagnostics stay typed and bounded because child-process errors can
// retain argv, environment, and output that must never enter the JSON result.
const aggregateDiagnosticMembers = new WeakMap();
const GENERATED_LAUNCH_AGENT_ENV_WRAPPER = `#!/bin/sh
set -eu
env_file="$1"
@@ -61,21 +72,222 @@ const DEPENDENCY_INPUT_RE =
/^(?:\.npmrc$|package\.json$|pnpm-lock\.yaml$|pnpm-workspace\.yaml$|patches\/)|(?:^|\/)package\.json$/u;
class UpdateInvariantError extends Error {
constructor(code, message, details) {
super(message);
constructor(code, message, details, options) {
super(message, options);
this.name = "UpdateInvariantError";
this.code = code;
this.details = details;
}
}
class UpdateCommandError extends Error {
constructor(operation, error) {
super(retainedErrorMessage(error), { cause: error });
this.name = "UpdateCommandError";
this.operation = operation;
const status = ownDataProperty(error, "status");
const signal = ownDataProperty(error, "signal");
if (Number.isInteger(status)) {
this.status = status;
}
if (typeof signal === "string" && /^SIG[A-Z0-9]+$/u.test(signal)) {
this.signal = signal;
}
}
}
/** Re-throw the original runtime value while exposing the Error contract to type-aware lint. */
function throwPreservingValue(value) {
throw /** @type {Error} */ (value);
}
function aggregateErrorWithCause(errors, message, cause) {
return new AggregateError(errors, message, { cause });
function ownDataProperty(value, key) {
if ((typeof value !== "object" && typeof value !== "function") || value === null) {
return undefined;
}
try {
const descriptor = Object.getOwnPropertyDescriptor(value, key);
return descriptor && "value" in descriptor ? descriptor.value : undefined;
} catch {
return undefined;
}
}
function failureMessage(error) {
if (error instanceof Error) {
if (error instanceof UpdateCommandError) {
return `${error.operation} failed`;
}
const status = ownDataProperty(error, "status");
const signal = ownDataProperty(error, "signal");
if (
Number.isInteger(status) ||
(typeof signal === "string" && /^SIG[A-Z0-9]+$/u.test(signal))
) {
return "external command failed";
}
const message = ownDataProperty(error, "message");
return typeof message === "string" ? message : error.name;
}
try {
return String(error);
} catch {
return "unknown updater failure";
}
}
function retainedErrorMessage(error) {
if (error instanceof Error) {
const message = ownDataProperty(error, "message");
return typeof message === "string" ? message : error.name;
}
try {
return String(error);
} catch {
return "unknown updater failure";
}
}
function aggregateErrorWithCause(members, message, cause) {
const error = new AggregateError(
members.map((member) => member.error),
message,
{ cause },
);
aggregateDiagnosticMembers.set(error, members);
return error;
}
function gatewayCliOperation(args) {
if (args[0] === "gateway" && args[1] === "call") {
if (args[2] === "gateway.suspend.prepare") {
return "gateway.suspend.prepare";
}
if (args[2] === "gateway.suspend.resume") {
return "gateway.suspend.resume";
}
return "gateway.call";
}
if (args[0] === "gateway" && args[1] === "status") {
return "gateway.status";
}
if (args[0] === "health") {
return "gateway.health";
}
return "gateway.cli";
}
function runUpdateCommand(runCommand, operation, command, args, checkout) {
try {
return runCommand(command, args, checkout);
} catch (error) {
if (
error instanceof UpdateInvariantError ||
error instanceof UpdateCommandError ||
error instanceof AggregateError
) {
throw error;
}
throw new UpdateCommandError(operation, error);
}
}
function formatInvariantDetails(details) {
const formatted = {};
for (const key of SAFE_INVARIANT_DETAIL_KEYS) {
const value = ownDataProperty(details, key);
if (typeof value === "boolean") {
formatted[key] = value;
} else if (key === "exitTimeoutSeconds" && Number.isInteger(value)) {
formatted[key] = value;
}
}
return Object.keys(formatted).length > 0 ? formatted : undefined;
}
function formatCommandDiagnostic(error, operation) {
const diagnostic = { kind: "command", operation };
const status = ownDataProperty(error, "status");
const signal = ownDataProperty(error, "signal");
if (Number.isInteger(status)) {
diagnostic.status = status;
}
if (typeof signal === "string" && /^SIG[A-Z0-9]+$/u.test(signal)) {
diagnostic.signal = signal;
}
return diagnostic;
}
function formatFailureDiagnostic(error, state, depth = 0) {
if (depth >= MAX_FAILURE_DIAGNOSTIC_DEPTH) {
return { kind: "truncated", reason: "depth_limit" };
}
if ((typeof error === "object" || typeof error === "function") && error !== null) {
if (state.seen.has(error)) {
return { kind: "truncated", reason: "cycle" };
}
state.seen.add(error);
}
if (error instanceof UpdateInvariantError) {
const details = formatInvariantDetails(error.details);
const cause = ownDataProperty(error, "cause");
return {
kind: "invariant",
code: error.code,
...(details ? { details } : {}),
...(cause === undefined ? {} : { cause: formatFailureDiagnostic(cause, state, depth + 1) }),
};
}
if (error instanceof UpdateCommandError) {
return formatCommandDiagnostic(error, error.operation);
}
if (error instanceof AggregateError) {
const rawErrors = ownDataProperty(error, "errors");
const errors = Array.isArray(rawErrors) ? rawErrors : [];
const members =
aggregateDiagnosticMembers.get(error) ??
errors.map((memberError, index) => ({
role: index === 0 ? "primary" : "secondary",
error: memberError,
}));
const limitedMembers = members.slice(0, MAX_FAILURE_DIAGNOSTIC_MEMBERS);
const diagnostic = {
kind: "aggregate",
members: limitedMembers.map((member) => ({
role: member.role,
error: formatFailureDiagnostic(member.error, state, depth + 1),
})),
};
const cause = ownDataProperty(error, "cause");
const causeMember = members.findIndex((member) => member.error === cause);
if (causeMember >= 0 && causeMember < limitedMembers.length) {
diagnostic.causeMember = causeMember;
}
if (members.length > limitedMembers.length) {
diagnostic.omittedMembers = members.length - limitedMembers.length;
}
return diagnostic;
}
const status = ownDataProperty(error, "status");
const signal = ownDataProperty(error, "signal");
if (Number.isInteger(status) || (typeof signal === "string" && /^SIG[A-Z0-9]+$/u.test(signal))) {
return formatCommandDiagnostic(error, "external_command");
}
return { kind: error instanceof Error ? "error" : "thrown_value" };
}
export function formatUpdateFailure(error) {
const code = error instanceof UpdateInvariantError ? error.code : "update_failed";
const message = failureMessage(error);
return {
schemaVersion: 1,
ok: false,
error: {
code,
message,
diagnostics: formatFailureDiagnostic(error, { seen: new Set() }),
},
};
}
function git(checkout, args, options = {}) {
@@ -777,6 +989,7 @@ export function resolveLaunchAgentExitTimeoutSeconds(value) {
throw new UpdateInvariantError(
"gateway_launchagent_failed",
`managed Gateway LaunchAgent ExitTimeOut=${value} prevents bounded stopped proof`,
{ exitTimeoutSeconds: value },
);
}
return Number.isInteger(value) && value > 0 ? value : DEFAULT_LAUNCH_AGENT_EXIT_TIMEOUT_SECONDS;
@@ -1092,7 +1305,10 @@ function prepareLaunchAgentEntrypointReplacement(deployment, entrypoint, options
restore();
} catch (restoreError) {
throw aggregateErrorWithCause(
[ownershipError, restoreError],
[
{ role: "primary", error: ownershipError },
{ role: "rollback", error: restoreError },
],
"System LaunchDaemon ownership changed during plist publication and the previous LaunchAgent could not be restored",
restoreError,
);
@@ -1296,6 +1512,8 @@ export function runBuiltGatewayCli(checkout, args, deployment, options = {}) {
stdio: ["ignore", "pipe", options.stderr ?? "inherit"],
timeout: options.timeoutMs ?? GATEWAY_CLI_TIMEOUT_MS,
});
} catch (error) {
throw new UpdateCommandError(gatewayCliOperation(args), error);
} finally {
rmSync(overlayPath, { force: true });
}
@@ -1333,7 +1551,9 @@ export function prepareGatewaySuspension(
} catch (error) {
throw new UpdateInvariantError(
"gateway_suspend_prepare_failed",
`could not atomically prepare Gateway maintenance: ${error instanceof Error ? error.message : String(error)}`,
"could not atomically prepare Gateway maintenance",
undefined,
{ cause: error },
);
}
if (result?.status === "ready" && typeof result.suspensionId === "string") {
@@ -1354,10 +1574,18 @@ function defaultResumeGatewaySuspension(checkout, suspensionId, deployment) {
function stopManagedGateway(runCommand, checkout, deployment) {
if (!deployment) {
runCommand(process.execPath, ["dist/index.js", "gateway", "stop"], checkout);
runUpdateCommand(
runCommand,
"gateway.stop",
process.execPath,
["dist/index.js", "gateway", "stop"],
checkout,
);
return;
}
runCommand(
runUpdateCommand(
runCommand,
"launchd.bootout",
"/bin/launchctl",
["bootout", `gui/${process.getuid()}/${deployment.label}`],
checkout,
@@ -1430,7 +1658,10 @@ function stopManagedGatewayAndProve(
throw proofError;
}
throw aggregateErrorWithCause(
[stopError, proofError],
[
{ role: "primary", error: stopError },
{ role: "proof", error: proofError },
],
"Gateway stop command failed and native stopped proof did not converge",
proofError,
);
@@ -1532,9 +1763,13 @@ function defaultProveGatewayStopped(checkout) {
}),
);
} catch (error) {
const commandError =
error instanceof UpdateCommandError ? error : new UpdateCommandError("gateway.status", error);
throw new UpdateInvariantError(
"gateway_stopped_proof_failed",
`could not inspect the managed Gateway after suspension failed: ${error instanceof Error ? error.message : String(error)}`,
"could not inspect the managed Gateway after suspension failed",
undefined,
{ cause: commandError },
);
}
const runtime = result?.service?.runtime;
@@ -1593,7 +1828,7 @@ function isOriginalMacBundle(bundlePath, originalStat) {
function runBuildWithPreservedMacApp(runCommand, checkout, sleep = defaultSleep) {
const appBundle = path.join(checkout, "dist/OpenClaw.app");
if (!existsSync(appBundle)) {
runCommand("pnpm", ["build"], checkout);
runUpdateCommand(runCommand, "build", "pnpm", ["build"], checkout);
return;
}
const appStat = lstatSync(appBundle);
@@ -1612,7 +1847,7 @@ function runBuildWithPreservedMacApp(runCommand, checkout, sleep = defaultSleep)
let buildFailed = false;
let buildError;
try {
runCommand("pnpm", ["build"], checkout);
runUpdateCommand(runCommand, "build", "pnpm", ["build"], checkout);
} catch (error) {
buildFailed = true;
buildError = error;
@@ -1685,7 +1920,13 @@ function restartGateway(
) {
assertExactBuild(checkout, expectedSha);
if (!deployment) {
runCommand("pnpm", ["openclaw", "gateway", "restart"], checkout);
runUpdateCommand(
runCommand,
"gateway.restart",
"pnpm",
["openclaw", "gateway", "restart"],
checkout,
);
return { processStartedAt: null, restartStartedAtMs: startedAtMs };
}
if (bootstrap) {
@@ -1700,7 +1941,9 @@ function restartGateway(
const assertOwnership =
options.assertNoSystemLaunchDaemonOwnership ?? assertNoSystemLaunchDaemonOwnership;
assertOwnership(deployment.label);
runCommand(
runUpdateCommand(
runCommand,
"gateway.restart",
deployment.executable,
[...deployment.invocationPrefix, "gateway", "restart"],
path.dirname(path.dirname(deployment.entrypoint)),
@@ -1724,8 +1967,20 @@ function bootstrapManagedGateway(runCommand, checkout, deployment, options = {})
const waitForProcess = options.waitForProcess ?? waitForManagedGatewayProcess;
const now = options.now ?? Date.now;
if (!options.startupTrace) {
runCommand("/bin/launchctl", ["enable", serviceTarget], checkout);
runCommand("/bin/launchctl", ["bootstrap", domain, deployment.plistPath], checkout);
runUpdateCommand(
runCommand,
"launchd.enable",
"/bin/launchctl",
["enable", serviceTarget],
checkout,
);
runUpdateCommand(
runCommand,
"launchd.bootstrap",
"/bin/launchctl",
["bootstrap", domain, deployment.plistPath],
checkout,
);
waitForProcess(deployment, options.sleep ?? defaultSleep);
return { processStartedAt: timestampAt(now) };
}
@@ -1736,10 +1991,28 @@ function bootstrapManagedGateway(runCommand, checkout, deployment, options = {})
const environmentRestore = armEnvironmentRestore(GATEWAY_STARTUP_TRACE_ENV, previousTraceValue);
let restartError;
let processStartedAt = null;
runCommand("/bin/launchctl", ["setenv", GATEWAY_STARTUP_TRACE_ENV, "1"], checkout);
runUpdateCommand(
runCommand,
"launchd.setenv",
"/bin/launchctl",
["setenv", GATEWAY_STARTUP_TRACE_ENV, "1"],
checkout,
);
try {
runCommand("/bin/launchctl", ["enable", serviceTarget], checkout);
runCommand("/bin/launchctl", ["bootstrap", domain, deployment.plistPath], checkout);
runUpdateCommand(
runCommand,
"launchd.enable",
"/bin/launchctl",
["enable", serviceTarget],
checkout,
);
runUpdateCommand(
runCommand,
"launchd.bootstrap",
"/bin/launchctl",
["bootstrap", domain, deployment.plistPath],
checkout,
);
waitForProcess(deployment, options.sleep ?? defaultSleep);
processStartedAt = timestampAt(now);
} catch (error) {
@@ -1748,7 +2021,9 @@ function bootstrapManagedGateway(runCommand, checkout, deployment, options = {})
try {
// The booted process already inherited the trace flag. Restore launchd's
// previous value immediately so later starts keep the host's normal config.
runCommand(
runUpdateCommand(
runCommand,
previousTraceValue === null ? "launchd.unsetenv" : "launchd.setenv",
"/bin/launchctl",
previousTraceValue === null
? ["unsetenv", GATEWAY_STARTUP_TRACE_ENV]
@@ -1758,7 +2033,10 @@ function bootstrapManagedGateway(runCommand, checkout, deployment, options = {})
} catch (cleanupError) {
if (restartError) {
throw aggregateErrorWithCause(
[restartError, cleanupError],
[
{ role: "primary", error: restartError },
{ role: "cleanup", error: cleanupError },
],
"Gateway restart failed and the one-shot startup trace environment could not be cleared",
cleanupError,
);
@@ -1988,7 +2266,9 @@ function verifyGatewayDeepRpc(runCommand, checkout, expectedSha, deployment, now
deployment,
);
} else {
runCommand(
runUpdateCommand(
runCommand,
"gateway.status",
"pnpm",
["openclaw", "gateway", "status", "--deep", "--require-rpc", "--json"],
checkout,
@@ -2015,7 +2295,13 @@ function readGatewayHealth(runCommand, checkout, deployment) {
}
return healthSummary;
}
runCommand("pnpm", ["openclaw", "health", "--verbose", "--json"], checkout);
runUpdateCommand(
runCommand,
"gateway.health",
"pnpm",
["openclaw", "health", "--verbose", "--json"],
checkout,
);
return null;
}
@@ -2553,7 +2839,13 @@ export function maintainMain(options, dependencies = {}) {
// clean source build cannot mutate its code. Build only to obtain
// an exact trusted client for the suspension RPC.
if (actions.dependencyInstall) {
runCommand("pnpm", ["install", "--frozen-lockfile"], update.checkout);
runUpdateCommand(
runCommand,
"dependencies.install",
"pnpm",
["install", "--frozen-lockfile"],
update.checkout,
);
controlDependenciesInstalled = true;
}
if (!actions.gatewayBuild) {
@@ -2581,12 +2873,15 @@ export function maintainMain(options, dependencies = {}) {
} catch (controlError) {
throw aggregateErrorWithCause(
[
new UpdateInvariantError(
"gateway_snapshot_control_unavailable",
"managed Gateway uses a snapshot but the source checkout has no exact trusted control build",
),
proofError,
controlError,
{
role: "context",
error: new UpdateInvariantError(
"gateway_snapshot_control_unavailable",
"managed Gateway uses a snapshot but the source checkout has no exact trusted control build",
),
},
{ role: "proof", error: proofError },
{ role: "primary", error: controlError },
],
"Gateway control is unavailable and the managed Gateway could not be proven stopped",
controlError,
@@ -2604,7 +2899,10 @@ export function maintainMain(options, dependencies = {}) {
};
} catch (proofError) {
throw aggregateErrorWithCause(
[prepareError, proofError],
[
{ role: "primary", error: prepareError },
{ role: "proof", error: proofError },
],
"Gateway suspension failed and the managed Gateway could not be proven stopped",
proofError,
);
@@ -2661,7 +2959,10 @@ export function maintainMain(options, dependencies = {}) {
);
} catch (resumeError) {
throw aggregateErrorWithCause(
[error, resumeError],
[
{ role: "primary", error },
{ role: "rollback", error: resumeError },
],
"Gateway stop failed and the prepared maintenance suspension could not be resumed",
resumeError,
);
@@ -2671,7 +2972,13 @@ export function maintainMain(options, dependencies = {}) {
}
try {
if (actions.dependencyInstall && !controlDependenciesInstalled) {
runCommand("pnpm", ["install", "--frozen-lockfile"], update.checkout);
runUpdateCommand(
runCommand,
"dependencies.install",
"pnpm",
["install", "--frozen-lockfile"],
update.checkout,
);
}
if (actions.gatewayBuild && !controlBuildPrepared) {
runBuildWithPreservedMacApp(runCommand, update.checkout, sleep);
@@ -2768,7 +3075,10 @@ export function maintainMain(options, dependencies = {}) {
waitForManagedGatewayReadiness(gatewayDeploymentBefore, probeMilestones, sleep);
} catch (recoveryError) {
throw aggregateErrorWithCause(
[error, recoveryError],
[
{ role: "primary", error },
{ role: "rollback", error: recoveryError },
],
"Gateway replacement failed and the previous managed service could not be restored",
recoveryError,
);
@@ -2845,7 +3155,9 @@ export function maintainMain(options, dependencies = {}) {
// The exact-SHA JS build above already produced dist/control-ui. Letting
// Mac packaging rebuild it can empty dist while the live app bundle is
// there, defeating the staged-swap guarantee.
runCommand(
runUpdateCommand(
runCommand,
"mac.restart",
"env",
[
"SKIP_TSC=1",
@@ -2932,9 +3244,7 @@ function main(argv = process.argv.slice(2)) {
try {
console.log(JSON.stringify(maintainMain(parseArgs(argv))));
} catch (error) {
const code = error instanceof UpdateInvariantError ? error.code : "update_failed";
const message = error instanceof Error ? error.message : String(error);
console.log(JSON.stringify({ schemaVersion: 1, ok: false, error: { code, message } }));
console.log(JSON.stringify(formatUpdateFailure(error)));
process.exitCode = 1;
}
}
+9
View File
@@ -189,6 +189,15 @@ declare module "*openclaw-live-updater/scripts/update-main.mjs" {
release?: () => void;
};
export function originMatches(remoteUrl: string): boolean;
export function formatUpdateFailure(error: unknown): {
schemaVersion: 1;
ok: false;
error: {
code: string;
message: string;
diagnostics: Record<string, unknown>;
};
};
export function isOwnedGatewayEntrypoint(
checkout: string,
home: string,
+314
View File
@@ -23,6 +23,7 @@ import {
assertNoSystemLaunchDaemonOwnership,
classifyActions,
findExactMacTarget,
formatUpdateFailure,
inspectBuildState,
isOwnedGatewayEntrypoint,
isGatewayProbeResponse,
@@ -340,6 +341,98 @@ describe("openclaw live updater", () => {
);
});
test("formats simple invariant diagnostics with allowlisted details", () => {
let failure: unknown;
try {
resolveLaunchAgentExitTimeoutSeconds(0);
} catch (error) {
failure = error;
}
expect(formatUpdateFailure(failure)).toEqual({
schemaVersion: 1,
ok: false,
error: {
code: "gateway_launchagent_failed",
message: "managed Gateway LaunchAgent ExitTimeOut=0 prevents bounded stopped proof",
diagnostics: {
kind: "invariant",
code: "gateway_launchagent_failed",
details: { exitTimeoutSeconds: 0 },
},
},
});
});
test("bounds recursive diagnostics and omits arbitrary error data", () => {
const commandError = Object.assign(new Error("nested-secret-message"), {
command: "/bin/private --token secret-command-token",
env: { SECRET: "secret-env-value" },
output: ["secret-output-value"],
status: 23,
stderr: "secret-stderr-value",
stdout: "secret-stdout-value",
});
const cyclic = new AggregateError([], "bounded aggregate");
cyclic.errors.push(
commandError,
cyclic,
...Array.from({ length: 8 }, () => new Error("extra")),
);
const formatted = formatUpdateFailure(cyclic);
expect(formatted.error.message).toBe("bounded aggregate");
const diagnostics = formatted.error.diagnostics as {
kind: string;
members: Array<Record<string, unknown>>;
omittedMembers: number;
};
expect(diagnostics).toMatchObject({
kind: "aggregate",
omittedMembers: 2,
});
expect(diagnostics.members).toHaveLength(8);
expect(diagnostics.members.slice(0, 2)).toEqual([
{
role: "primary",
error: { kind: "command", operation: "external_command", status: 23 },
},
{
role: "secondary",
error: { kind: "truncated", reason: "cycle" },
},
]);
const serialized = JSON.stringify(formatted);
expect(serialized).not.toContain("secret-");
expect(serialized).not.toContain("/bin/private");
let nested: unknown = new Error("leaf");
for (let depth = 0; depth < 5; depth += 1) {
nested = new AggregateError([nested], `level-${depth}`);
}
expect(JSON.stringify(formatUpdateFailure(nested))).toContain('"reason":"depth_limit"');
});
test("formats hostile non-Error thrown values without reading arbitrary fields", () => {
const failure = {
secret: "secret-object-value",
toString() {
throw new Error("secret-to-string-value");
},
};
const formatted = formatUpdateFailure(failure);
expect(formatted).toMatchObject({
ok: false,
error: {
code: "update_failed",
message: "unknown updater failure",
diagnostics: { kind: "thrown_value" },
},
});
expect(JSON.stringify(formatted)).not.toContain("secret-");
});
test("fails closed on same-label system LaunchDaemon ownership", () => {
const missing = { status: 113, stdout: "", stderr: "Could not find service" };
expect(() =>
@@ -1543,6 +1636,93 @@ console.log(JSON.stringify({ ok: true, channels: {} }));
]);
});
test("preserves typed suspension command diagnostics when stopped proof also fails", () => {
const { root, mirror } = makeFixture();
mkdirSync(path.join(mirror, "node_modules"));
const configPath = path.join(root, "openclaw.json");
const entrypoint = path.join(mirror, "dist/index.js");
mkdirSync(path.dirname(entrypoint), { recursive: true });
writeFileSync(configPath, "{}\n");
writeFileSync(
entrypoint,
'if (process.argv.includes("status")) process.exit(29); process.stderr.write("secret suspension stderr\\n"); process.exit(23);\n',
);
const deployment = {
configPath,
entrypoint,
executable: process.execPath,
invocationPrefix: [entrypoint],
port: 18789,
runtime: process.execPath,
serviceEnvironment: {},
wrapperPath: null,
};
let failure: unknown;
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
try {
Object.defineProperty(process, "platform", { value: "linux" });
maintainFixture(
{ checkout: mirror, remote: "origin", lockPath: path.join(root, "maintenance.lock") },
{
prepareGatewaySuspension: (checkout: string) =>
prepareGatewaySuspension(
checkout,
() =>
runBuiltGatewayCli(
checkout,
["gateway", "call", "gateway.suspend.prepare"],
deployment,
{ stderr: "pipe" },
),
deployment,
),
proveGatewayStopped: undefined,
},
);
} catch (error) {
failure = error;
} finally {
if (platformDescriptor) {
Object.defineProperty(process, "platform", platformDescriptor);
}
}
const formatted = formatUpdateFailure(failure);
expect(formatted.error.diagnostics).toEqual({
kind: "aggregate",
members: [
{
role: "primary",
error: {
kind: "invariant",
code: "gateway_suspend_prepare_failed",
cause: {
kind: "command",
operation: "gateway.suspend.prepare",
status: 23,
},
},
},
{
role: "proof",
error: {
kind: "invariant",
code: "gateway_stopped_proof_failed",
cause: {
kind: "command",
operation: "gateway.status",
status: 29,
},
},
},
],
causeMember: 1,
});
expect(JSON.stringify(formatted)).not.toContain("secret suspension stderr");
expect(JSON.stringify(formatted)).not.toContain(entrypoint);
});
test("preserves the signed Mac bundle while a Gateway build replaces dist", () => {
const { root, mirror } = makeFixture();
mkdirSync(path.join(mirror, "node_modules"));
@@ -1594,6 +1774,42 @@ console.log(JSON.stringify({ ok: true, channels: {} }));
expect(readFileSync(appMarker, "utf8")).toBe("signed\n");
});
test("reports a standalone command operation without command output", () => {
const { root, mirror } = makeFixture();
mkdirSync(path.join(mirror, "node_modules"));
let failure: unknown;
try {
maintainFixture(
{ checkout: mirror, remote: "origin", lockPath: path.join(root, "maintenance.lock") },
{
runCommand(command: string, args: string[]) {
if (command === "pnpm" && args[0] === "build") {
throw Object.assign(new Error("secret build message"), {
status: 17,
stderr: "secret build stderr",
stdout: "secret build stdout",
});
}
},
},
);
} catch (error) {
failure = error;
}
const formatted = formatUpdateFailure(failure);
expect(formatted.error.message).toBe("build failed");
expect(formatted.error.diagnostics).toEqual({
kind: "command",
operation: "build",
status: 17,
});
expect(JSON.stringify(formatted)).not.toContain("secret build stderr");
expect(JSON.stringify(formatted)).not.toContain("secret build stdout");
expect(JSON.stringify(formatted)).not.toContain("secret build message");
});
test("accepts a delayed external restore of the exact preserved Mac bundle", () => {
const { root, mirror } = makeFixture();
mkdirSync(path.join(mirror, "node_modules"));
@@ -1977,6 +2193,82 @@ console.log(JSON.stringify({ ok: true, channels: {} }));
]);
});
test("reports primary invariant and rollback command diagnostics", () => {
const { root, mirror } = makeFixture();
mkdirSync(path.join(mirror, "node_modules"));
const source = path.join(mirror, "dist/index.js");
const plistPath = path.join(root, "ai.openclaw.gateway.plist");
writeFileSync(plistPath, "plist\n", { mode: 0o600 });
let failure: unknown;
try {
maintainFixture(
{ checkout: mirror, remote: "origin", lockPath: path.join(root, "maintenance.lock") },
{
inspectGatewayDeployment: () => ({
configPath: path.join(root, "openclaw.json"),
entrypoint: source,
entrypointIndex: 1,
executable: process.execPath,
invocationPrefix: [source],
label: "ai.openclaw.gateway",
plistPath,
port: 18789,
runtime: process.execPath,
}),
runCommand(command: string, args: string[]) {
if (command === "pnpm" && args[0] === "build") {
resolveLaunchAgentExitTimeoutSeconds(0);
}
if (command === "/bin/launchctl" && args[0] === "bootstrap") {
throw Object.assign(new Error("secret rollback command message"), {
status: 23,
stderr: "secret rollback stderr",
stdout: "secret rollback stdout",
});
}
},
},
);
} catch (error) {
failure = error;
}
const formatted = formatUpdateFailure(failure);
expect(formatted).toMatchObject({
schemaVersion: 1,
ok: false,
error: {
code: "update_failed",
message:
"Gateway replacement failed and the previous managed service could not be restored",
diagnostics: {
kind: "aggregate",
causeMember: 1,
members: [
{
role: "primary",
error: {
kind: "invariant",
code: "gateway_launchagent_failed",
details: { exitTimeoutSeconds: 0 },
},
},
{
role: "rollback",
error: {
kind: "command",
operation: "launchd.bootstrap",
status: 23,
},
},
],
},
},
});
expect(JSON.stringify(formatted)).not.toContain("secret rollback");
});
test("resumes suspension when system ownership appears before bootout", () => {
const { root, mirror, seed } = makeFixture({ includeSeed: true });
mkdirSync(path.join(mirror, "node_modules"));
@@ -2349,6 +2641,28 @@ console.log(JSON.stringify({ ok: true, channels: {} }));
expect(result.stderr).toContain("child-output");
});
test("keeps failed CLI stdout as one additive machine-readable JSON object", () => {
const result = spawnSync(process.execPath, [script, "--definitely-invalid"], {
encoding: "utf8",
});
expect(result.status).toBe(1);
expect(result.stdout.trim().split("\n")).toHaveLength(1);
expect(JSON.parse(result.stdout)).toEqual({
schemaVersion: 1,
ok: false,
error: {
code: "invalid_argument",
message: "unknown argument: --definitely-invalid",
diagnostics: {
kind: "invariant",
code: "invalid_argument",
},
},
});
expect(result.stderr).toBe("");
});
test("does not restart Gateway when build provenance misses the exact SHA", () => {
const { root, mirror } = makeFixture();
mkdirSync(path.join(mirror, "node_modules"));