mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix: preserve live updater failure diagnostics (#119096)
This commit is contained in:
@@ -48,6 +48,17 @@ const GATEWAY_PROCESS_START_RETRY_DELAY_MS = 250;
|
||||
const GATEWAY_SUSPEND_TIMEOUT_MS = 10_000;
|
||||
const GATEWAY_STARTUP_TRACE_ENV = "OPENCLAW_GATEWAY_STARTUP_TRACE";
|
||||
const SYSTEM_LAUNCH_DAEMON_DIR = "/Library/LaunchDaemons";
|
||||
const MAX_FAILURE_DIAGNOSTIC_DEPTH = 4;
|
||||
const MAX_FAILURE_DIAGNOSTIC_MEMBERS = 8;
|
||||
const SAFE_INVARIANT_DETAIL_KEYS = [
|
||||
"exitTimeoutSeconds",
|
||||
"listenerClosed",
|
||||
"processExited",
|
||||
"serviceBootedOut",
|
||||
];
|
||||
// CLI diagnostics stay typed and bounded because child-process errors can
|
||||
// retain argv, environment, and output that must never enter the JSON result.
|
||||
const aggregateDiagnosticMembers = new WeakMap();
|
||||
const GENERATED_LAUNCH_AGENT_ENV_WRAPPER = `#!/bin/sh
|
||||
set -eu
|
||||
env_file="$1"
|
||||
@@ -61,21 +72,222 @@ const DEPENDENCY_INPUT_RE =
|
||||
/^(?:\.npmrc$|package\.json$|pnpm-lock\.yaml$|pnpm-workspace\.yaml$|patches\/)|(?:^|\/)package\.json$/u;
|
||||
|
||||
class UpdateInvariantError extends Error {
|
||||
constructor(code, message, details) {
|
||||
super(message);
|
||||
constructor(code, message, details, options) {
|
||||
super(message, options);
|
||||
this.name = "UpdateInvariantError";
|
||||
this.code = code;
|
||||
this.details = details;
|
||||
}
|
||||
}
|
||||
|
||||
class UpdateCommandError extends Error {
|
||||
constructor(operation, error) {
|
||||
super(retainedErrorMessage(error), { cause: error });
|
||||
this.name = "UpdateCommandError";
|
||||
this.operation = operation;
|
||||
const status = ownDataProperty(error, "status");
|
||||
const signal = ownDataProperty(error, "signal");
|
||||
if (Number.isInteger(status)) {
|
||||
this.status = status;
|
||||
}
|
||||
if (typeof signal === "string" && /^SIG[A-Z0-9]+$/u.test(signal)) {
|
||||
this.signal = signal;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-throw the original runtime value while exposing the Error contract to type-aware lint. */
|
||||
function throwPreservingValue(value) {
|
||||
throw /** @type {Error} */ (value);
|
||||
}
|
||||
|
||||
function aggregateErrorWithCause(errors, message, cause) {
|
||||
return new AggregateError(errors, message, { cause });
|
||||
function ownDataProperty(value, key) {
|
||||
if ((typeof value !== "object" && typeof value !== "function") || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
||||
return descriptor && "value" in descriptor ? descriptor.value : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function failureMessage(error) {
|
||||
if (error instanceof Error) {
|
||||
if (error instanceof UpdateCommandError) {
|
||||
return `${error.operation} failed`;
|
||||
}
|
||||
const status = ownDataProperty(error, "status");
|
||||
const signal = ownDataProperty(error, "signal");
|
||||
if (
|
||||
Number.isInteger(status) ||
|
||||
(typeof signal === "string" && /^SIG[A-Z0-9]+$/u.test(signal))
|
||||
) {
|
||||
return "external command failed";
|
||||
}
|
||||
const message = ownDataProperty(error, "message");
|
||||
return typeof message === "string" ? message : error.name;
|
||||
}
|
||||
try {
|
||||
return String(error);
|
||||
} catch {
|
||||
return "unknown updater failure";
|
||||
}
|
||||
}
|
||||
|
||||
function retainedErrorMessage(error) {
|
||||
if (error instanceof Error) {
|
||||
const message = ownDataProperty(error, "message");
|
||||
return typeof message === "string" ? message : error.name;
|
||||
}
|
||||
try {
|
||||
return String(error);
|
||||
} catch {
|
||||
return "unknown updater failure";
|
||||
}
|
||||
}
|
||||
|
||||
function aggregateErrorWithCause(members, message, cause) {
|
||||
const error = new AggregateError(
|
||||
members.map((member) => member.error),
|
||||
message,
|
||||
{ cause },
|
||||
);
|
||||
aggregateDiagnosticMembers.set(error, members);
|
||||
return error;
|
||||
}
|
||||
|
||||
function gatewayCliOperation(args) {
|
||||
if (args[0] === "gateway" && args[1] === "call") {
|
||||
if (args[2] === "gateway.suspend.prepare") {
|
||||
return "gateway.suspend.prepare";
|
||||
}
|
||||
if (args[2] === "gateway.suspend.resume") {
|
||||
return "gateway.suspend.resume";
|
||||
}
|
||||
return "gateway.call";
|
||||
}
|
||||
if (args[0] === "gateway" && args[1] === "status") {
|
||||
return "gateway.status";
|
||||
}
|
||||
if (args[0] === "health") {
|
||||
return "gateway.health";
|
||||
}
|
||||
return "gateway.cli";
|
||||
}
|
||||
|
||||
function runUpdateCommand(runCommand, operation, command, args, checkout) {
|
||||
try {
|
||||
return runCommand(command, args, checkout);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof UpdateInvariantError ||
|
||||
error instanceof UpdateCommandError ||
|
||||
error instanceof AggregateError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new UpdateCommandError(operation, error);
|
||||
}
|
||||
}
|
||||
|
||||
function formatInvariantDetails(details) {
|
||||
const formatted = {};
|
||||
for (const key of SAFE_INVARIANT_DETAIL_KEYS) {
|
||||
const value = ownDataProperty(details, key);
|
||||
if (typeof value === "boolean") {
|
||||
formatted[key] = value;
|
||||
} else if (key === "exitTimeoutSeconds" && Number.isInteger(value)) {
|
||||
formatted[key] = value;
|
||||
}
|
||||
}
|
||||
return Object.keys(formatted).length > 0 ? formatted : undefined;
|
||||
}
|
||||
|
||||
function formatCommandDiagnostic(error, operation) {
|
||||
const diagnostic = { kind: "command", operation };
|
||||
const status = ownDataProperty(error, "status");
|
||||
const signal = ownDataProperty(error, "signal");
|
||||
if (Number.isInteger(status)) {
|
||||
diagnostic.status = status;
|
||||
}
|
||||
if (typeof signal === "string" && /^SIG[A-Z0-9]+$/u.test(signal)) {
|
||||
diagnostic.signal = signal;
|
||||
}
|
||||
return diagnostic;
|
||||
}
|
||||
|
||||
function formatFailureDiagnostic(error, state, depth = 0) {
|
||||
if (depth >= MAX_FAILURE_DIAGNOSTIC_DEPTH) {
|
||||
return { kind: "truncated", reason: "depth_limit" };
|
||||
}
|
||||
if ((typeof error === "object" || typeof error === "function") && error !== null) {
|
||||
if (state.seen.has(error)) {
|
||||
return { kind: "truncated", reason: "cycle" };
|
||||
}
|
||||
state.seen.add(error);
|
||||
}
|
||||
if (error instanceof UpdateInvariantError) {
|
||||
const details = formatInvariantDetails(error.details);
|
||||
const cause = ownDataProperty(error, "cause");
|
||||
return {
|
||||
kind: "invariant",
|
||||
code: error.code,
|
||||
...(details ? { details } : {}),
|
||||
...(cause === undefined ? {} : { cause: formatFailureDiagnostic(cause, state, depth + 1) }),
|
||||
};
|
||||
}
|
||||
if (error instanceof UpdateCommandError) {
|
||||
return formatCommandDiagnostic(error, error.operation);
|
||||
}
|
||||
if (error instanceof AggregateError) {
|
||||
const rawErrors = ownDataProperty(error, "errors");
|
||||
const errors = Array.isArray(rawErrors) ? rawErrors : [];
|
||||
const members =
|
||||
aggregateDiagnosticMembers.get(error) ??
|
||||
errors.map((memberError, index) => ({
|
||||
role: index === 0 ? "primary" : "secondary",
|
||||
error: memberError,
|
||||
}));
|
||||
const limitedMembers = members.slice(0, MAX_FAILURE_DIAGNOSTIC_MEMBERS);
|
||||
const diagnostic = {
|
||||
kind: "aggregate",
|
||||
members: limitedMembers.map((member) => ({
|
||||
role: member.role,
|
||||
error: formatFailureDiagnostic(member.error, state, depth + 1),
|
||||
})),
|
||||
};
|
||||
const cause = ownDataProperty(error, "cause");
|
||||
const causeMember = members.findIndex((member) => member.error === cause);
|
||||
if (causeMember >= 0 && causeMember < limitedMembers.length) {
|
||||
diagnostic.causeMember = causeMember;
|
||||
}
|
||||
if (members.length > limitedMembers.length) {
|
||||
diagnostic.omittedMembers = members.length - limitedMembers.length;
|
||||
}
|
||||
return diagnostic;
|
||||
}
|
||||
const status = ownDataProperty(error, "status");
|
||||
const signal = ownDataProperty(error, "signal");
|
||||
if (Number.isInteger(status) || (typeof signal === "string" && /^SIG[A-Z0-9]+$/u.test(signal))) {
|
||||
return formatCommandDiagnostic(error, "external_command");
|
||||
}
|
||||
return { kind: error instanceof Error ? "error" : "thrown_value" };
|
||||
}
|
||||
|
||||
export function formatUpdateFailure(error) {
|
||||
const code = error instanceof UpdateInvariantError ? error.code : "update_failed";
|
||||
const message = failureMessage(error);
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
ok: false,
|
||||
error: {
|
||||
code,
|
||||
message,
|
||||
diagnostics: formatFailureDiagnostic(error, { seen: new Set() }),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function git(checkout, args, options = {}) {
|
||||
@@ -777,6 +989,7 @@ export function resolveLaunchAgentExitTimeoutSeconds(value) {
|
||||
throw new UpdateInvariantError(
|
||||
"gateway_launchagent_failed",
|
||||
`managed Gateway LaunchAgent ExitTimeOut=${value} prevents bounded stopped proof`,
|
||||
{ exitTimeoutSeconds: value },
|
||||
);
|
||||
}
|
||||
return Number.isInteger(value) && value > 0 ? value : DEFAULT_LAUNCH_AGENT_EXIT_TIMEOUT_SECONDS;
|
||||
@@ -1092,7 +1305,10 @@ function prepareLaunchAgentEntrypointReplacement(deployment, entrypoint, options
|
||||
restore();
|
||||
} catch (restoreError) {
|
||||
throw aggregateErrorWithCause(
|
||||
[ownershipError, restoreError],
|
||||
[
|
||||
{ role: "primary", error: ownershipError },
|
||||
{ role: "rollback", error: restoreError },
|
||||
],
|
||||
"System LaunchDaemon ownership changed during plist publication and the previous LaunchAgent could not be restored",
|
||||
restoreError,
|
||||
);
|
||||
@@ -1296,6 +1512,8 @@ export function runBuiltGatewayCli(checkout, args, deployment, options = {}) {
|
||||
stdio: ["ignore", "pipe", options.stderr ?? "inherit"],
|
||||
timeout: options.timeoutMs ?? GATEWAY_CLI_TIMEOUT_MS,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new UpdateCommandError(gatewayCliOperation(args), error);
|
||||
} finally {
|
||||
rmSync(overlayPath, { force: true });
|
||||
}
|
||||
@@ -1333,7 +1551,9 @@ export function prepareGatewaySuspension(
|
||||
} catch (error) {
|
||||
throw new UpdateInvariantError(
|
||||
"gateway_suspend_prepare_failed",
|
||||
`could not atomically prepare Gateway maintenance: ${error instanceof Error ? error.message : String(error)}`,
|
||||
"could not atomically prepare Gateway maintenance",
|
||||
undefined,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
if (result?.status === "ready" && typeof result.suspensionId === "string") {
|
||||
@@ -1354,10 +1574,18 @@ function defaultResumeGatewaySuspension(checkout, suspensionId, deployment) {
|
||||
|
||||
function stopManagedGateway(runCommand, checkout, deployment) {
|
||||
if (!deployment) {
|
||||
runCommand(process.execPath, ["dist/index.js", "gateway", "stop"], checkout);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"gateway.stop",
|
||||
process.execPath,
|
||||
["dist/index.js", "gateway", "stop"],
|
||||
checkout,
|
||||
);
|
||||
return;
|
||||
}
|
||||
runCommand(
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.bootout",
|
||||
"/bin/launchctl",
|
||||
["bootout", `gui/${process.getuid()}/${deployment.label}`],
|
||||
checkout,
|
||||
@@ -1430,7 +1658,10 @@ function stopManagedGatewayAndProve(
|
||||
throw proofError;
|
||||
}
|
||||
throw aggregateErrorWithCause(
|
||||
[stopError, proofError],
|
||||
[
|
||||
{ role: "primary", error: stopError },
|
||||
{ role: "proof", error: proofError },
|
||||
],
|
||||
"Gateway stop command failed and native stopped proof did not converge",
|
||||
proofError,
|
||||
);
|
||||
@@ -1532,9 +1763,13 @@ function defaultProveGatewayStopped(checkout) {
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
const commandError =
|
||||
error instanceof UpdateCommandError ? error : new UpdateCommandError("gateway.status", error);
|
||||
throw new UpdateInvariantError(
|
||||
"gateway_stopped_proof_failed",
|
||||
`could not inspect the managed Gateway after suspension failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
"could not inspect the managed Gateway after suspension failed",
|
||||
undefined,
|
||||
{ cause: commandError },
|
||||
);
|
||||
}
|
||||
const runtime = result?.service?.runtime;
|
||||
@@ -1593,7 +1828,7 @@ function isOriginalMacBundle(bundlePath, originalStat) {
|
||||
function runBuildWithPreservedMacApp(runCommand, checkout, sleep = defaultSleep) {
|
||||
const appBundle = path.join(checkout, "dist/OpenClaw.app");
|
||||
if (!existsSync(appBundle)) {
|
||||
runCommand("pnpm", ["build"], checkout);
|
||||
runUpdateCommand(runCommand, "build", "pnpm", ["build"], checkout);
|
||||
return;
|
||||
}
|
||||
const appStat = lstatSync(appBundle);
|
||||
@@ -1612,7 +1847,7 @@ function runBuildWithPreservedMacApp(runCommand, checkout, sleep = defaultSleep)
|
||||
let buildFailed = false;
|
||||
let buildError;
|
||||
try {
|
||||
runCommand("pnpm", ["build"], checkout);
|
||||
runUpdateCommand(runCommand, "build", "pnpm", ["build"], checkout);
|
||||
} catch (error) {
|
||||
buildFailed = true;
|
||||
buildError = error;
|
||||
@@ -1685,7 +1920,13 @@ function restartGateway(
|
||||
) {
|
||||
assertExactBuild(checkout, expectedSha);
|
||||
if (!deployment) {
|
||||
runCommand("pnpm", ["openclaw", "gateway", "restart"], checkout);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"gateway.restart",
|
||||
"pnpm",
|
||||
["openclaw", "gateway", "restart"],
|
||||
checkout,
|
||||
);
|
||||
return { processStartedAt: null, restartStartedAtMs: startedAtMs };
|
||||
}
|
||||
if (bootstrap) {
|
||||
@@ -1700,7 +1941,9 @@ function restartGateway(
|
||||
const assertOwnership =
|
||||
options.assertNoSystemLaunchDaemonOwnership ?? assertNoSystemLaunchDaemonOwnership;
|
||||
assertOwnership(deployment.label);
|
||||
runCommand(
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"gateway.restart",
|
||||
deployment.executable,
|
||||
[...deployment.invocationPrefix, "gateway", "restart"],
|
||||
path.dirname(path.dirname(deployment.entrypoint)),
|
||||
@@ -1724,8 +1967,20 @@ function bootstrapManagedGateway(runCommand, checkout, deployment, options = {})
|
||||
const waitForProcess = options.waitForProcess ?? waitForManagedGatewayProcess;
|
||||
const now = options.now ?? Date.now;
|
||||
if (!options.startupTrace) {
|
||||
runCommand("/bin/launchctl", ["enable", serviceTarget], checkout);
|
||||
runCommand("/bin/launchctl", ["bootstrap", domain, deployment.plistPath], checkout);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.enable",
|
||||
"/bin/launchctl",
|
||||
["enable", serviceTarget],
|
||||
checkout,
|
||||
);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.bootstrap",
|
||||
"/bin/launchctl",
|
||||
["bootstrap", domain, deployment.plistPath],
|
||||
checkout,
|
||||
);
|
||||
waitForProcess(deployment, options.sleep ?? defaultSleep);
|
||||
return { processStartedAt: timestampAt(now) };
|
||||
}
|
||||
@@ -1736,10 +1991,28 @@ function bootstrapManagedGateway(runCommand, checkout, deployment, options = {})
|
||||
const environmentRestore = armEnvironmentRestore(GATEWAY_STARTUP_TRACE_ENV, previousTraceValue);
|
||||
let restartError;
|
||||
let processStartedAt = null;
|
||||
runCommand("/bin/launchctl", ["setenv", GATEWAY_STARTUP_TRACE_ENV, "1"], checkout);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.setenv",
|
||||
"/bin/launchctl",
|
||||
["setenv", GATEWAY_STARTUP_TRACE_ENV, "1"],
|
||||
checkout,
|
||||
);
|
||||
try {
|
||||
runCommand("/bin/launchctl", ["enable", serviceTarget], checkout);
|
||||
runCommand("/bin/launchctl", ["bootstrap", domain, deployment.plistPath], checkout);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.enable",
|
||||
"/bin/launchctl",
|
||||
["enable", serviceTarget],
|
||||
checkout,
|
||||
);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.bootstrap",
|
||||
"/bin/launchctl",
|
||||
["bootstrap", domain, deployment.plistPath],
|
||||
checkout,
|
||||
);
|
||||
waitForProcess(deployment, options.sleep ?? defaultSleep);
|
||||
processStartedAt = timestampAt(now);
|
||||
} catch (error) {
|
||||
@@ -1748,7 +2021,9 @@ function bootstrapManagedGateway(runCommand, checkout, deployment, options = {})
|
||||
try {
|
||||
// The booted process already inherited the trace flag. Restore launchd's
|
||||
// previous value immediately so later starts keep the host's normal config.
|
||||
runCommand(
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
previousTraceValue === null ? "launchd.unsetenv" : "launchd.setenv",
|
||||
"/bin/launchctl",
|
||||
previousTraceValue === null
|
||||
? ["unsetenv", GATEWAY_STARTUP_TRACE_ENV]
|
||||
@@ -1758,7 +2033,10 @@ function bootstrapManagedGateway(runCommand, checkout, deployment, options = {})
|
||||
} catch (cleanupError) {
|
||||
if (restartError) {
|
||||
throw aggregateErrorWithCause(
|
||||
[restartError, cleanupError],
|
||||
[
|
||||
{ role: "primary", error: restartError },
|
||||
{ role: "cleanup", error: cleanupError },
|
||||
],
|
||||
"Gateway restart failed and the one-shot startup trace environment could not be cleared",
|
||||
cleanupError,
|
||||
);
|
||||
@@ -1988,7 +2266,9 @@ function verifyGatewayDeepRpc(runCommand, checkout, expectedSha, deployment, now
|
||||
deployment,
|
||||
);
|
||||
} else {
|
||||
runCommand(
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"gateway.status",
|
||||
"pnpm",
|
||||
["openclaw", "gateway", "status", "--deep", "--require-rpc", "--json"],
|
||||
checkout,
|
||||
@@ -2015,7 +2295,13 @@ function readGatewayHealth(runCommand, checkout, deployment) {
|
||||
}
|
||||
return healthSummary;
|
||||
}
|
||||
runCommand("pnpm", ["openclaw", "health", "--verbose", "--json"], checkout);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"gateway.health",
|
||||
"pnpm",
|
||||
["openclaw", "health", "--verbose", "--json"],
|
||||
checkout,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -2553,7 +2839,13 @@ export function maintainMain(options, dependencies = {}) {
|
||||
// clean source build cannot mutate its code. Build only to obtain
|
||||
// an exact trusted client for the suspension RPC.
|
||||
if (actions.dependencyInstall) {
|
||||
runCommand("pnpm", ["install", "--frozen-lockfile"], update.checkout);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"dependencies.install",
|
||||
"pnpm",
|
||||
["install", "--frozen-lockfile"],
|
||||
update.checkout,
|
||||
);
|
||||
controlDependenciesInstalled = true;
|
||||
}
|
||||
if (!actions.gatewayBuild) {
|
||||
@@ -2581,12 +2873,15 @@ export function maintainMain(options, dependencies = {}) {
|
||||
} catch (controlError) {
|
||||
throw aggregateErrorWithCause(
|
||||
[
|
||||
new UpdateInvariantError(
|
||||
"gateway_snapshot_control_unavailable",
|
||||
"managed Gateway uses a snapshot but the source checkout has no exact trusted control build",
|
||||
),
|
||||
proofError,
|
||||
controlError,
|
||||
{
|
||||
role: "context",
|
||||
error: new UpdateInvariantError(
|
||||
"gateway_snapshot_control_unavailable",
|
||||
"managed Gateway uses a snapshot but the source checkout has no exact trusted control build",
|
||||
),
|
||||
},
|
||||
{ role: "proof", error: proofError },
|
||||
{ role: "primary", error: controlError },
|
||||
],
|
||||
"Gateway control is unavailable and the managed Gateway could not be proven stopped",
|
||||
controlError,
|
||||
@@ -2604,7 +2899,10 @@ export function maintainMain(options, dependencies = {}) {
|
||||
};
|
||||
} catch (proofError) {
|
||||
throw aggregateErrorWithCause(
|
||||
[prepareError, proofError],
|
||||
[
|
||||
{ role: "primary", error: prepareError },
|
||||
{ role: "proof", error: proofError },
|
||||
],
|
||||
"Gateway suspension failed and the managed Gateway could not be proven stopped",
|
||||
proofError,
|
||||
);
|
||||
@@ -2661,7 +2959,10 @@ export function maintainMain(options, dependencies = {}) {
|
||||
);
|
||||
} catch (resumeError) {
|
||||
throw aggregateErrorWithCause(
|
||||
[error, resumeError],
|
||||
[
|
||||
{ role: "primary", error },
|
||||
{ role: "rollback", error: resumeError },
|
||||
],
|
||||
"Gateway stop failed and the prepared maintenance suspension could not be resumed",
|
||||
resumeError,
|
||||
);
|
||||
@@ -2671,7 +2972,13 @@ export function maintainMain(options, dependencies = {}) {
|
||||
}
|
||||
try {
|
||||
if (actions.dependencyInstall && !controlDependenciesInstalled) {
|
||||
runCommand("pnpm", ["install", "--frozen-lockfile"], update.checkout);
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"dependencies.install",
|
||||
"pnpm",
|
||||
["install", "--frozen-lockfile"],
|
||||
update.checkout,
|
||||
);
|
||||
}
|
||||
if (actions.gatewayBuild && !controlBuildPrepared) {
|
||||
runBuildWithPreservedMacApp(runCommand, update.checkout, sleep);
|
||||
@@ -2768,7 +3075,10 @@ export function maintainMain(options, dependencies = {}) {
|
||||
waitForManagedGatewayReadiness(gatewayDeploymentBefore, probeMilestones, sleep);
|
||||
} catch (recoveryError) {
|
||||
throw aggregateErrorWithCause(
|
||||
[error, recoveryError],
|
||||
[
|
||||
{ role: "primary", error },
|
||||
{ role: "rollback", error: recoveryError },
|
||||
],
|
||||
"Gateway replacement failed and the previous managed service could not be restored",
|
||||
recoveryError,
|
||||
);
|
||||
@@ -2845,7 +3155,9 @@ export function maintainMain(options, dependencies = {}) {
|
||||
// The exact-SHA JS build above already produced dist/control-ui. Letting
|
||||
// Mac packaging rebuild it can empty dist while the live app bundle is
|
||||
// there, defeating the staged-swap guarantee.
|
||||
runCommand(
|
||||
runUpdateCommand(
|
||||
runCommand,
|
||||
"mac.restart",
|
||||
"env",
|
||||
[
|
||||
"SKIP_TSC=1",
|
||||
@@ -2932,9 +3244,7 @@ function main(argv = process.argv.slice(2)) {
|
||||
try {
|
||||
console.log(JSON.stringify(maintainMain(parseArgs(argv))));
|
||||
} catch (error) {
|
||||
const code = error instanceof UpdateInvariantError ? error.code : "update_failed";
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.log(JSON.stringify({ schemaVersion: 1, ok: false, error: { code, message } }));
|
||||
console.log(JSON.stringify(formatUpdateFailure(error)));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+9
@@ -189,6 +189,15 @@ declare module "*openclaw-live-updater/scripts/update-main.mjs" {
|
||||
release?: () => void;
|
||||
};
|
||||
export function originMatches(remoteUrl: string): boolean;
|
||||
export function formatUpdateFailure(error: unknown): {
|
||||
schemaVersion: 1;
|
||||
ok: false;
|
||||
error: {
|
||||
code: string;
|
||||
message: string;
|
||||
diagnostics: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
export function isOwnedGatewayEntrypoint(
|
||||
checkout: string,
|
||||
home: string,
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
assertNoSystemLaunchDaemonOwnership,
|
||||
classifyActions,
|
||||
findExactMacTarget,
|
||||
formatUpdateFailure,
|
||||
inspectBuildState,
|
||||
isOwnedGatewayEntrypoint,
|
||||
isGatewayProbeResponse,
|
||||
@@ -340,6 +341,98 @@ describe("openclaw live updater", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("formats simple invariant diagnostics with allowlisted details", () => {
|
||||
let failure: unknown;
|
||||
try {
|
||||
resolveLaunchAgentExitTimeoutSeconds(0);
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
|
||||
expect(formatUpdateFailure(failure)).toEqual({
|
||||
schemaVersion: 1,
|
||||
ok: false,
|
||||
error: {
|
||||
code: "gateway_launchagent_failed",
|
||||
message: "managed Gateway LaunchAgent ExitTimeOut=0 prevents bounded stopped proof",
|
||||
diagnostics: {
|
||||
kind: "invariant",
|
||||
code: "gateway_launchagent_failed",
|
||||
details: { exitTimeoutSeconds: 0 },
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("bounds recursive diagnostics and omits arbitrary error data", () => {
|
||||
const commandError = Object.assign(new Error("nested-secret-message"), {
|
||||
command: "/bin/private --token secret-command-token",
|
||||
env: { SECRET: "secret-env-value" },
|
||||
output: ["secret-output-value"],
|
||||
status: 23,
|
||||
stderr: "secret-stderr-value",
|
||||
stdout: "secret-stdout-value",
|
||||
});
|
||||
const cyclic = new AggregateError([], "bounded aggregate");
|
||||
cyclic.errors.push(
|
||||
commandError,
|
||||
cyclic,
|
||||
...Array.from({ length: 8 }, () => new Error("extra")),
|
||||
);
|
||||
|
||||
const formatted = formatUpdateFailure(cyclic);
|
||||
expect(formatted.error.message).toBe("bounded aggregate");
|
||||
const diagnostics = formatted.error.diagnostics as {
|
||||
kind: string;
|
||||
members: Array<Record<string, unknown>>;
|
||||
omittedMembers: number;
|
||||
};
|
||||
expect(diagnostics).toMatchObject({
|
||||
kind: "aggregate",
|
||||
omittedMembers: 2,
|
||||
});
|
||||
expect(diagnostics.members).toHaveLength(8);
|
||||
expect(diagnostics.members.slice(0, 2)).toEqual([
|
||||
{
|
||||
role: "primary",
|
||||
error: { kind: "command", operation: "external_command", status: 23 },
|
||||
},
|
||||
{
|
||||
role: "secondary",
|
||||
error: { kind: "truncated", reason: "cycle" },
|
||||
},
|
||||
]);
|
||||
const serialized = JSON.stringify(formatted);
|
||||
expect(serialized).not.toContain("secret-");
|
||||
expect(serialized).not.toContain("/bin/private");
|
||||
|
||||
let nested: unknown = new Error("leaf");
|
||||
for (let depth = 0; depth < 5; depth += 1) {
|
||||
nested = new AggregateError([nested], `level-${depth}`);
|
||||
}
|
||||
expect(JSON.stringify(formatUpdateFailure(nested))).toContain('"reason":"depth_limit"');
|
||||
});
|
||||
|
||||
test("formats hostile non-Error thrown values without reading arbitrary fields", () => {
|
||||
const failure = {
|
||||
secret: "secret-object-value",
|
||||
toString() {
|
||||
throw new Error("secret-to-string-value");
|
||||
},
|
||||
};
|
||||
|
||||
const formatted = formatUpdateFailure(failure);
|
||||
expect(formatted).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
code: "update_failed",
|
||||
message: "unknown updater failure",
|
||||
diagnostics: { kind: "thrown_value" },
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(formatted)).not.toContain("secret-");
|
||||
});
|
||||
|
||||
test("fails closed on same-label system LaunchDaemon ownership", () => {
|
||||
const missing = { status: 113, stdout: "", stderr: "Could not find service" };
|
||||
expect(() =>
|
||||
@@ -1543,6 +1636,93 @@ console.log(JSON.stringify({ ok: true, channels: {} }));
|
||||
]);
|
||||
});
|
||||
|
||||
test("preserves typed suspension command diagnostics when stopped proof also fails", () => {
|
||||
const { root, mirror } = makeFixture();
|
||||
mkdirSync(path.join(mirror, "node_modules"));
|
||||
const configPath = path.join(root, "openclaw.json");
|
||||
const entrypoint = path.join(mirror, "dist/index.js");
|
||||
mkdirSync(path.dirname(entrypoint), { recursive: true });
|
||||
writeFileSync(configPath, "{}\n");
|
||||
writeFileSync(
|
||||
entrypoint,
|
||||
'if (process.argv.includes("status")) process.exit(29); process.stderr.write("secret suspension stderr\\n"); process.exit(23);\n',
|
||||
);
|
||||
const deployment = {
|
||||
configPath,
|
||||
entrypoint,
|
||||
executable: process.execPath,
|
||||
invocationPrefix: [entrypoint],
|
||||
port: 18789,
|
||||
runtime: process.execPath,
|
||||
serviceEnvironment: {},
|
||||
wrapperPath: null,
|
||||
};
|
||||
let failure: unknown;
|
||||
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
|
||||
try {
|
||||
Object.defineProperty(process, "platform", { value: "linux" });
|
||||
maintainFixture(
|
||||
{ checkout: mirror, remote: "origin", lockPath: path.join(root, "maintenance.lock") },
|
||||
{
|
||||
prepareGatewaySuspension: (checkout: string) =>
|
||||
prepareGatewaySuspension(
|
||||
checkout,
|
||||
() =>
|
||||
runBuiltGatewayCli(
|
||||
checkout,
|
||||
["gateway", "call", "gateway.suspend.prepare"],
|
||||
deployment,
|
||||
{ stderr: "pipe" },
|
||||
),
|
||||
deployment,
|
||||
),
|
||||
proveGatewayStopped: undefined,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
} finally {
|
||||
if (platformDescriptor) {
|
||||
Object.defineProperty(process, "platform", platformDescriptor);
|
||||
}
|
||||
}
|
||||
|
||||
const formatted = formatUpdateFailure(failure);
|
||||
expect(formatted.error.diagnostics).toEqual({
|
||||
kind: "aggregate",
|
||||
members: [
|
||||
{
|
||||
role: "primary",
|
||||
error: {
|
||||
kind: "invariant",
|
||||
code: "gateway_suspend_prepare_failed",
|
||||
cause: {
|
||||
kind: "command",
|
||||
operation: "gateway.suspend.prepare",
|
||||
status: 23,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
role: "proof",
|
||||
error: {
|
||||
kind: "invariant",
|
||||
code: "gateway_stopped_proof_failed",
|
||||
cause: {
|
||||
kind: "command",
|
||||
operation: "gateway.status",
|
||||
status: 29,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
causeMember: 1,
|
||||
});
|
||||
expect(JSON.stringify(formatted)).not.toContain("secret suspension stderr");
|
||||
expect(JSON.stringify(formatted)).not.toContain(entrypoint);
|
||||
});
|
||||
|
||||
test("preserves the signed Mac bundle while a Gateway build replaces dist", () => {
|
||||
const { root, mirror } = makeFixture();
|
||||
mkdirSync(path.join(mirror, "node_modules"));
|
||||
@@ -1594,6 +1774,42 @@ console.log(JSON.stringify({ ok: true, channels: {} }));
|
||||
expect(readFileSync(appMarker, "utf8")).toBe("signed\n");
|
||||
});
|
||||
|
||||
test("reports a standalone command operation without command output", () => {
|
||||
const { root, mirror } = makeFixture();
|
||||
mkdirSync(path.join(mirror, "node_modules"));
|
||||
let failure: unknown;
|
||||
|
||||
try {
|
||||
maintainFixture(
|
||||
{ checkout: mirror, remote: "origin", lockPath: path.join(root, "maintenance.lock") },
|
||||
{
|
||||
runCommand(command: string, args: string[]) {
|
||||
if (command === "pnpm" && args[0] === "build") {
|
||||
throw Object.assign(new Error("secret build message"), {
|
||||
status: 17,
|
||||
stderr: "secret build stderr",
|
||||
stdout: "secret build stdout",
|
||||
});
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
|
||||
const formatted = formatUpdateFailure(failure);
|
||||
expect(formatted.error.message).toBe("build failed");
|
||||
expect(formatted.error.diagnostics).toEqual({
|
||||
kind: "command",
|
||||
operation: "build",
|
||||
status: 17,
|
||||
});
|
||||
expect(JSON.stringify(formatted)).not.toContain("secret build stderr");
|
||||
expect(JSON.stringify(formatted)).not.toContain("secret build stdout");
|
||||
expect(JSON.stringify(formatted)).not.toContain("secret build message");
|
||||
});
|
||||
|
||||
test("accepts a delayed external restore of the exact preserved Mac bundle", () => {
|
||||
const { root, mirror } = makeFixture();
|
||||
mkdirSync(path.join(mirror, "node_modules"));
|
||||
@@ -1977,6 +2193,82 @@ console.log(JSON.stringify({ ok: true, channels: {} }));
|
||||
]);
|
||||
});
|
||||
|
||||
test("reports primary invariant and rollback command diagnostics", () => {
|
||||
const { root, mirror } = makeFixture();
|
||||
mkdirSync(path.join(mirror, "node_modules"));
|
||||
const source = path.join(mirror, "dist/index.js");
|
||||
const plistPath = path.join(root, "ai.openclaw.gateway.plist");
|
||||
writeFileSync(plistPath, "plist\n", { mode: 0o600 });
|
||||
let failure: unknown;
|
||||
|
||||
try {
|
||||
maintainFixture(
|
||||
{ checkout: mirror, remote: "origin", lockPath: path.join(root, "maintenance.lock") },
|
||||
{
|
||||
inspectGatewayDeployment: () => ({
|
||||
configPath: path.join(root, "openclaw.json"),
|
||||
entrypoint: source,
|
||||
entrypointIndex: 1,
|
||||
executable: process.execPath,
|
||||
invocationPrefix: [source],
|
||||
label: "ai.openclaw.gateway",
|
||||
plistPath,
|
||||
port: 18789,
|
||||
runtime: process.execPath,
|
||||
}),
|
||||
runCommand(command: string, args: string[]) {
|
||||
if (command === "pnpm" && args[0] === "build") {
|
||||
resolveLaunchAgentExitTimeoutSeconds(0);
|
||||
}
|
||||
if (command === "/bin/launchctl" && args[0] === "bootstrap") {
|
||||
throw Object.assign(new Error("secret rollback command message"), {
|
||||
status: 23,
|
||||
stderr: "secret rollback stderr",
|
||||
stdout: "secret rollback stdout",
|
||||
});
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
|
||||
const formatted = formatUpdateFailure(failure);
|
||||
expect(formatted).toMatchObject({
|
||||
schemaVersion: 1,
|
||||
ok: false,
|
||||
error: {
|
||||
code: "update_failed",
|
||||
message:
|
||||
"Gateway replacement failed and the previous managed service could not be restored",
|
||||
diagnostics: {
|
||||
kind: "aggregate",
|
||||
causeMember: 1,
|
||||
members: [
|
||||
{
|
||||
role: "primary",
|
||||
error: {
|
||||
kind: "invariant",
|
||||
code: "gateway_launchagent_failed",
|
||||
details: { exitTimeoutSeconds: 0 },
|
||||
},
|
||||
},
|
||||
{
|
||||
role: "rollback",
|
||||
error: {
|
||||
kind: "command",
|
||||
operation: "launchd.bootstrap",
|
||||
status: 23,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(formatted)).not.toContain("secret rollback");
|
||||
});
|
||||
|
||||
test("resumes suspension when system ownership appears before bootout", () => {
|
||||
const { root, mirror, seed } = makeFixture({ includeSeed: true });
|
||||
mkdirSync(path.join(mirror, "node_modules"));
|
||||
@@ -2349,6 +2641,28 @@ console.log(JSON.stringify({ ok: true, channels: {} }));
|
||||
expect(result.stderr).toContain("child-output");
|
||||
});
|
||||
|
||||
test("keeps failed CLI stdout as one additive machine-readable JSON object", () => {
|
||||
const result = spawnSync(process.execPath, [script, "--definitely-invalid"], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stdout.trim().split("\n")).toHaveLength(1);
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
schemaVersion: 1,
|
||||
ok: false,
|
||||
error: {
|
||||
code: "invalid_argument",
|
||||
message: "unknown argument: --definitely-invalid",
|
||||
diagnostics: {
|
||||
kind: "invariant",
|
||||
code: "invalid_argument",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result.stderr).toBe("");
|
||||
});
|
||||
|
||||
test("does not restart Gateway when build provenance misses the exact SHA", () => {
|
||||
const { root, mirror } = makeFixture();
|
||||
mkdirSync(path.join(mirror, "node_modules"));
|
||||
|
||||
Reference in New Issue
Block a user