From 2ed24246182f0a02677ffc9abe745927f610f7d5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 01:41:27 -0700 Subject: [PATCH] feat: harden live gateway and mac restarts (#104314) * feat: audit gateway logs after live restarts * fix: keep mac app running during replacement build --- .agents/skills/openclaw-live-updater/SKILL.md | 1 + .../scripts/update-main.mjs | 74 ++++++++++++++++++- scripts/restart-mac.sh | 18 +++-- test/scripts/openclaw-live-updater.test.ts | 69 ++++++++++++++++- test/scripts/restart-mac.test.ts | 27 +++++-- 5 files changed, 176 insertions(+), 13 deletions(-) diff --git a/.agents/skills/openclaw-live-updater/SKILL.md b/.agents/skills/openclaw-live-updater/SKILL.md index 373be8c99ad7..3550857abba1 100644 --- a/.agents/skills/openclaw-live-updater/SKILL.md +++ b/.agents/skills/openclaw-live-updater/SKILL.md @@ -32,6 +32,7 @@ Keep `/Users/steipete/openclaw` a read-only-to-the-agent deployment mirror: clea - A dependency-input change, absent `node_modules`, or missing/invalid build provenance first runs `pnpm install --frozen-lockfile`. - `pnpm build` must leave both canonical stamp heads and `dist/build-info.json.commit` equal to post-update `afterSha`; any missing/mismatched stamp or required artifact blocks restart. - Only after exact-SHA build proof may it restart the managed Gateway and require `gateway status --deep --require-rpc --json` plus `health --verbose --json`. + - After every managed restart, query Gateway logs through RPC, restrict the audit to entries emitted since that restart began, report warning summaries, and fail the pass on any error/fatal entry. Never accept supervisor or RPC health without this restart-window log audit. Treat supervisor state alone as insufficient. If build or proof fails, leave the new mirror head intact and retry the stale/missing build on the next heartbeat; never run the old `dist` against new source. diff --git a/.agents/skills/openclaw-live-updater/scripts/update-main.mjs b/.agents/skills/openclaw-live-updater/scripts/update-main.mjs index f1ad28501a4a..b1c0ce99267c 100644 --- a/.agents/skills/openclaw-live-updater/scripts/update-main.mjs +++ b/.agents/skills/openclaw-live-updater/scripts/update-main.mjs @@ -524,7 +524,9 @@ function assertExactBuild(checkout, expectedSha) { function restartGateway(runCommand, checkout, expectedSha) { assertExactBuild(checkout, expectedSha); + const startedAtMs = Date.now(); runCommand("pnpm", ["openclaw", "gateway", "restart"], checkout); + return startedAtMs; } function verifyGateway(runCommand, checkout, expectedSha) { @@ -537,6 +539,69 @@ function verifyGateway(runCommand, checkout, expectedSha) { runCommand("pnpm", ["openclaw", "health", "--verbose", "--json"], checkout); } +function summarizeGatewayLogEntry(entry) { + return { + time: entry.time, + level: entry.level, + subsystem: entry.subsystem ?? null, + message: String(entry.message ?? "").slice(0, 500), + }; +} + +export function parseGatewayLogAudit(output, sinceMs) { + const entries = output + .split("\n") + .filter(Boolean) + .flatMap((line) => { + try { + const entry = JSON.parse(line); + const timestamp = Date.parse(entry.time ?? ""); + return entry.type === "log" && Number.isFinite(timestamp) && timestamp >= sinceMs + ? [entry] + : []; + } catch { + return []; + } + }); + const errors = entries + .filter((entry) => entry.level === "error" || entry.level === "fatal") + .map(summarizeGatewayLogEntry); + const warnings = entries.filter((entry) => entry.level === "warn").map(summarizeGatewayLogEntry); + return { + entries: entries.length, + errorCount: errors.length, + warningCount: warnings.length, + errors: errors.slice(0, 20), + warnings: warnings.slice(0, 20), + }; +} + +function defaultAuditGatewayLogs(checkout, sinceMs) { + const output = execFileSync( + process.execPath, + [ + "openclaw.mjs", + "logs", + "--json", + "--limit", + "1000", + "--max-bytes", + "1000000", + "--timeout", + "10000", + ], + { cwd: checkout, encoding: "utf8", maxBuffer: 4 * 1024 * 1024 }, + ); + const audit = parseGatewayLogAudit(output, sinceMs); + if (audit.errorCount > 0) { + throw new UpdateInvariantError( + "gateway_restart_log_errors", + `Gateway emitted ${audit.errorCount} error/fatal log entries after restart: ${JSON.stringify(audit.errors.slice(0, 5))}`, + ); + } + return audit; +} + export function findExactMacTarget(processes, executable) { const target = processes .split("\n") @@ -601,6 +666,8 @@ export function maintainMain(options, dependencies = {}) { } const runCommand = dependencies.runCommand ?? defaultRunCommand; const verifyMacTarget = dependencies.verifyMacTarget ?? defaultVerifyMacTarget; + const auditGatewayLogs = dependencies.auditGatewayLogs ?? defaultAuditGatewayLogs; + let gatewayLogAudit = null; let queuedMacState = null; if (actions.macAppRebuild) { queuedMacState = { @@ -619,16 +686,18 @@ export function maintainMain(options, dependencies = {}) { if (actions.gatewayBuild) { runCommand("pnpm", ["build"], update.checkout); assertExactBuild(update.checkout, update.afterSha); - restartGateway(runCommand, update.checkout, update.afterSha); + const restartStartedAt = restartGateway(runCommand, update.checkout, update.afterSha); verifyGateway(runCommand, update.checkout, update.afterSha); + gatewayLogAudit = auditGatewayLogs(update.checkout, restartStartedAt); } else { try { verifyGateway(runCommand, update.checkout, update.afterSha); } catch { actions.gatewayRestart = true; actions.gatewaySelfHeal = true; - restartGateway(runCommand, update.checkout, update.afterSha); + const restartStartedAt = restartGateway(runCommand, update.checkout, update.afterSha); verifyGateway(runCommand, update.checkout, update.afterSha); + gatewayLogAudit = auditGatewayLogs(update.checkout, restartStartedAt); } } if (actions.macAppRebuild) { @@ -664,6 +733,7 @@ export function maintainMain(options, dependencies = {}) { buildBefore, buildChangedPaths, actions, + ...(gatewayLogAudit ? { gatewayLogAudit } : {}), ...(maintenanceState.macTarget ? { macTarget: maintenanceState.macTarget } : {}), }; } finally { diff --git a/scripts/restart-mac.sh b/scripts/restart-mac.sh index b34035bc3bc3..d59c1fcba733 100755 --- a/scripts/restart-mac.sh +++ b/scripts/restart-mac.sh @@ -269,15 +269,13 @@ stop_launch_agent() { launchctl bootout gui/"$UID"/ai.openclaw.mac 2>/dev/null || true } -# 1) Stop only the process set selected by the requested mode. +# 1) Validate the process set selected by the requested mode. Target-only keeps +# the current managed app alive while the replacement builds and signs. if [[ "$TARGET_ONLY" -eq 1 ]]; then if [[ -n "$(foreign_openclaw_process_pids)" ]]; then fail "Another OpenClaw app or test process is active; target-only restart deferred" fi - log "==> Killing managed installed and exact target OpenClaw instances" - if ! kill_managed_openclaw; then - fail "Managed OpenClaw instances did not exit after cleanup attempts" - fi + log "==> Keeping managed OpenClaw running while the replacement builds" else stop_launch_agent log "==> Killing existing OpenClaw instances" @@ -378,6 +376,16 @@ if [[ "$ATTACH_ONLY" -eq 1 ]]; then ATTACH_ONLY_ARGS+=(--args --attach-only) fi +if [[ "$TARGET_ONLY" -eq 1 ]]; then + if [[ -n "$(foreign_openclaw_process_pids)" ]]; then + fail "Another OpenClaw app or test process appeared during build; target-only restart deferred" + fi + log "==> Switching managed installed and exact target OpenClaw instances" + if ! kill_managed_openclaw; then + fail "Managed OpenClaw instances did not exit after cleanup attempts" + fi +fi + # 4) Launch the installed app in the foreground so the menu bar extra appears. # LaunchServices can inherit a huge environment from this shell (secrets, prompt vars, etc.). # That can cause launchd spawn failures and is undesirable for a GUI app anyway. diff --git a/test/scripts/openclaw-live-updater.test.ts b/test/scripts/openclaw-live-updater.test.ts index 34199822acbd..9c8409bf4083 100644 --- a/test/scripts/openclaw-live-updater.test.ts +++ b/test/scripts/openclaw-live-updater.test.ts @@ -5,6 +5,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + realpathSync, renameSync, rmSync, symlinkSync, @@ -21,6 +22,7 @@ import { inspectBuildState, maintainMain, originMatches, + parseGatewayLogAudit, } from "../../.agents/skills/openclaw-live-updater/scripts/update-main.mjs"; import { BUILD_STAMP_FILE, @@ -48,7 +50,17 @@ function maintainFixture( options: Record, dependencies: Record = {}, ) { - return maintainMain(options, { fetchMain: fetchFixtureMain, ...dependencies }); + return maintainMain(options, { + fetchMain: fetchFixtureMain, + auditGatewayLogs: () => ({ + entries: 0, + errorCount: 0, + warningCount: 0, + errors: [], + warnings: [], + }), + ...dependencies, + }); } function makeFixture() { @@ -72,6 +84,7 @@ function makeFixture() { const canonicalOrigin = "https://github.com/openclaw/openclaw.git"; git(mirror, "remote", "set-url", "origin", canonicalOrigin); fixtureOrigins.set(mirror, origin); + fixtureOrigins.set(realpathSync(mirror), origin); return { root, mirror, origin, seed }; } @@ -116,6 +129,53 @@ function fakeCommands(mirror: string) { } describe("openclaw live updater", () => { + test("audits only error and warning logs emitted after Gateway restart", () => { + const output = [ + { type: "meta", file: "/tmp/openclaw.log" }, + { type: "log", time: "2026-07-11T08:00:00.000Z", level: "error", message: "old" }, + { type: "log", time: "2026-07-11T08:00:02.000Z", level: "info", message: "ready" }, + { + type: "log", + time: "2026-07-11T08:00:03.000Z", + level: "warn", + subsystem: "gateway", + message: "degraded", + }, + { + type: "log", + time: "2026-07-11T08:00:04.000Z", + level: "fatal", + subsystem: "gateway", + message: "failed", + }, + { type: "notice", message: "done" }, + ] + .map((entry) => JSON.stringify(entry)) + .join("\n"); + + expect(parseGatewayLogAudit(output, Date.parse("2026-07-11T08:00:02.000Z"))).toEqual({ + entries: 3, + errorCount: 1, + warningCount: 1, + errors: [ + { + time: "2026-07-11T08:00:04.000Z", + level: "fatal", + subsystem: "gateway", + message: "failed", + }, + ], + warnings: [ + { + time: "2026-07-11T08:00:03.000Z", + level: "warn", + subsystem: "gateway", + message: "degraded", + }, + ], + }); + }); + test("accepts supported OpenClaw GitHub origins", () => { expect(originMatches("https://github.com/openclaw/openclaw.git")).toBe(true); expect(originMatches("git@github.com:openclaw/openclaw.git")).toBe(true); @@ -275,6 +335,13 @@ describe("openclaw live updater", () => { macAppRebuild: true, macUiVerification: true, }); + expect(output.gatewayLogAudit).toEqual({ + entries: 0, + errorCount: 0, + warningCount: 0, + errors: [], + warnings: [], + }); expect(commands.calls).toEqual([ "pnpm install --frozen-lockfile", "pnpm build", diff --git a/test/scripts/restart-mac.test.ts b/test/scripts/restart-mac.test.ts index 92c1a4450521..5e16871cc43c 100644 --- a/test/scripts/restart-mac.test.ts +++ b/test/scripts/restart-mac.test.ts @@ -372,20 +372,37 @@ describe("scripts/restart-mac.sh", () => { it("target-only mode refuses foreign app processes without broad cleanup", () => { const script = readFileSync(restartScriptPath, "utf8"); - const targetBlock = script.slice( + const initialTargetBlock = script.slice( script.indexOf('if [[ "$TARGET_ONLY" -eq 1 ]]; then', script.indexOf("# 1)")), script.indexOf("else", script.indexOf("# 1)")), ); + const switchTargetBlock = script.slice( + script.indexOf('if [[ "$TARGET_ONLY" -eq 1 ]]; then', script.indexOf("ATTACH_ONLY_ARGS")), + script.indexOf("# 4) Launch"), + ); - expect(targetBlock).toContain("foreign_openclaw_process_pids"); - expect(targetBlock).toContain("kill_managed_openclaw"); - expect(targetBlock).not.toContain("stop_launch_agent"); - expect(targetBlock).not.toContain("kill_all_openclaw"); + expect(initialTargetBlock).toContain("foreign_openclaw_process_pids"); + expect(initialTargetBlock).not.toContain("kill_managed_openclaw"); + expect(initialTargetBlock).not.toContain("stop_launch_agent"); + expect(initialTargetBlock).not.toContain("kill_all_openclaw"); + expect(switchTargetBlock).toContain("foreign_openclaw_process_pids"); + expect(switchTargetBlock).toContain("kill_managed_openclaw"); expect(script).toContain('[[ "${executable}" == "${TARGET_EXECUTABLE}" ]] && continue'); expect(script).toContain('process_pids_for_executable "${TARGET_EXECUTABLE}"'); expect(script).toContain("target-only restart deferred"); }); + it("keeps the managed app alive until the signed replacement is ready", () => { + const script = readFileSync(restartScriptPath, "utf8"); + const packageIndex = script.indexOf('run_step "package app"'); + const switchIndex = script.indexOf('log "==> Switching managed installed'); + const launchIndex = script.indexOf('run_step "launch app"'); + + expect(packageIndex).toBeGreaterThan(-1); + expect(switchIndex).toBeGreaterThan(packageIndex); + expect(launchIndex).toBeGreaterThan(switchIndex); + }); + it("escalates only exact managed app processes when graceful shutdown stalls", () => { const script = readFileSync(restartScriptPath, "utf8"); const managedKillBlock = script.slice(