diff --git a/docs/gateway/restart-recovery.md b/docs/gateway/restart-recovery.md index 305b6403c84d..0a6d7a649d01 100644 --- a/docs/gateway/restart-recovery.md +++ b/docs/gateway/restart-recovery.md @@ -186,7 +186,8 @@ restart handling continues. resumes on a later boot after the unclean-boot window drains. Gateway logs look like: `channel autostart suppressed by crash-loop breaker; refusing automatic -start for … Use channels.start to override.` +start for … Start a channel manually with: openclaw gateway call +channels.start --params '{"channel":""}'` Operator recovery SOP: diff --git a/src/cli/gateway-cli/run.option-collisions.test.ts b/src/cli/gateway-cli/run.option-collisions.test.ts index acf47bccec64..230fea49e6be 100644 --- a/src/cli/gateway-cli/run.option-collisions.test.ts +++ b/src/cli/gateway-cli/run.option-collisions.test.ts @@ -96,6 +96,7 @@ const writeDiagnosticStabilityBundleForFailureSync = vi.fn((_reason: string, _er path: "/tmp/openclaw-stability.json", })); const bootLifecycle = vi.hoisted(() => ({ + manualChannelStartHint: `Start a channel manually with: openclaw gateway call channels.start --params '{"channel":""}'`, decisions: [] as Array<{ tripped: boolean; uncleanBoots: number; @@ -297,6 +298,7 @@ vi.mock("../../logging/diagnostic-stability-bundle.js", () => ({ vi.mock("../../infra/gateway-boot-lifecycle.js", () => ({ GATEWAY_CRASH_LOOP_BREAKER_REASON: "gateway.crash_loop_breaker", + formatGatewayCrashLoopManualChannelStartHint: () => bootLifecycle.manualChannelStartHint, GATEWAY_CRASH_LOOP_RECOVERED_REASON: "gateway.crash_loop_recovered", inspectGatewayCrashLoopBreaker: (env?: NodeJS.ProcessEnv, nowMs?: number) => bootLifecycle.inspect(env, nowMs), @@ -449,7 +451,7 @@ describe("gateway run option collisions", () => { return callArg(startGatewayServer, index, 1) as { auth?: { mode?: string; token?: string; password?: string }; bind?: string; - channelAutostartSuppression?: { reason?: string }; + channelAutostartSuppression?: { reason?: string; message?: string }; ambientEnvTriggers?: "allow" | "suppress"; startupConfigSnapshotRead?: { snapshot?: Record }; startupStartedAt?: number; @@ -1570,6 +1572,9 @@ describe("gateway run option collisions", () => { expect(gatewayStartOptions(0).channelAutostartSuppression).toMatchObject({ reason: "crash-loop-breaker", }); + expect(gatewayStartOptions(0).channelAutostartSuppression?.message).toContain( + bootLifecycle.manualChannelStartHint, + ); expect(gatewayStartOptions(1).channelAutostartSuppression).toBeUndefined(); expect(gatewayLogMessages.some((message) => message.includes("breaker recovered"))).toBe(true); }); diff --git a/src/cli/gateway-cli/run.ts b/src/cli/gateway-cli/run.ts index a60f324bf739..128f8ed2dc4b 100644 --- a/src/cli/gateway-cli/run.ts +++ b/src/cli/gateway-cli/run.ts @@ -39,6 +39,7 @@ import { formatErrorMessage } from "../../infra/errors.js"; import { completeGatewayBootLifecycle, GATEWAY_CRASH_LOOP_BREAKER_REASON, + formatGatewayCrashLoopManualChannelStartHint, GATEWAY_CRASH_LOOP_RECOVERED_REASON, inspectGatewayCrashLoopBreaker, recordGatewayBootStart, @@ -1136,7 +1137,7 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt } const message = `gateway restart-loop breaker tripped: ${crashLoopDecision.uncleanBoots} unclean boot(s) within ${crashLoopDecision.windowMs}ms; ` + - "suppressing channel/provider account auto-start. Inspect the stability bundle and fix the startup crash before restarting the service."; + `suppressing channel/provider account auto-start. Inspect the stability bundle and fix the startup crash before restarting the service. ${formatGatewayCrashLoopManualChannelStartHint()}`; channelAutostartSuppression = { reason: "crash-loop-breaker", message }; gatewayLog.error(message); if (crashLoopDecision.shouldWriteStabilityBundle) { diff --git a/src/gateway/server-channels.ts b/src/gateway/server-channels.ts index f636f9959bd5..d400ca707448 100644 --- a/src/gateway/server-channels.ts +++ b/src/gateway/server-channels.ts @@ -12,6 +12,7 @@ import { startChannelApprovalHandlerBootstrap } from "../infra/approval-handler- import { type BackoffPolicy, sleepWithAbort } from "../infra/backoff.js"; import { createTaskScopedChannelRuntime } from "../infra/channel-runtime-context.js"; import { formatErrorMessage } from "../infra/errors.js"; +import { formatGatewayCrashLoopManualChannelStartHint } from "../infra/gateway-boot-lifecycle.js"; import { resetDirectoryCache } from "../infra/outbound/target-resolver.js"; import { createSubsystemLogger, @@ -481,7 +482,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage // config reloads can undo the crash-loop breaker while operators inspect. const suffix = accountId ? ` account ${accountId}` : ""; ensureChannelLog(channelId).warn?.( - `channel autostart suppressed by crash-loop breaker; refusing automatic start for ${channelId}${suffix}. Use channels.start to override.`, + `channel autostart suppressed by crash-loop breaker; refusing automatic start for ${channelId}${suffix}. ${formatGatewayCrashLoopManualChannelStartHint({ channelId, ...(accountId ? { accountId } : {}) })}`, ); for (const id of accountIds) { setStoppedRuntime(channelId, id, { diff --git a/src/infra/gateway-boot-lifecycle.test.ts b/src/infra/gateway-boot-lifecycle.test.ts index a26bcc4e2e02..0dd79e6b8867 100644 --- a/src/infra/gateway-boot-lifecycle.test.ts +++ b/src/infra/gateway-boot-lifecycle.test.ts @@ -12,6 +12,7 @@ import { GATEWAY_CRASH_LOOP_BREAKER_REASON, GATEWAY_CRASH_LOOP_RECOVERED_REASON, completeGatewayBootLifecycle, + formatGatewayCrashLoopManualChannelStartHint, inspectGatewayCrashLoopBreaker, recordGatewayBootStart, } from "./gateway-boot-lifecycle.js"; @@ -220,3 +221,25 @@ describe("gateway crash-loop breaker", () => { expect(rows).not.toContain("old"); }); }); + +describe("formatGatewayCrashLoopManualChannelStartHint", () => { + it("uses a placeholder when no channel is known", () => { + expect(formatGatewayCrashLoopManualChannelStartHint()).toContain( + `--params '{"channel":""}'`, + ); + }); + + it("names the channel being suppressed", () => { + expect(formatGatewayCrashLoopManualChannelStartHint({ channelId: "telegram" })).toContain( + `--params '{"channel":"telegram"}'`, + ); + }); + + // Suppression is reported per account; omitting accountId would tell operators to run a command + // that starts the channel's default account instead of the one the warning named. + it("carries the account when suppression is account-scoped", () => { + expect( + formatGatewayCrashLoopManualChannelStartHint({ channelId: "telegram", accountId: "work" }), + ).toContain(`--params '{"channel":"telegram","accountId":"work"}'`); + }); +}); diff --git a/src/infra/gateway-boot-lifecycle.ts b/src/infra/gateway-boot-lifecycle.ts index 0a99d704f893..07ea2fb30e82 100644 --- a/src/infra/gateway-boot-lifecycle.ts +++ b/src/infra/gateway-boot-lifecycle.ts @@ -22,6 +22,24 @@ const GATEWAY_BOOT_LOOP_WINDOW_MS = 5 * 60_000; const GATEWAY_BOOT_LIFECYCLE_RETENTION_MS = 24 * 60 * 60_000; export const GATEWAY_CRASH_LOOP_BREAKER_REASON = "gateway.crash_loop_breaker"; export const GATEWAY_CRASH_LOOP_RECOVERED_REASON = "gateway.crash_loop_recovered"; +/** + * The breaker never self-clears within its window, so every operator-facing surface must name the + * manual override command instead of the internal RPC name. Account-scoped suppression must carry + * its accountId: `channels.start` resolves an omitted account to the channel default, so a hint + * without it would start a different account than the one the message named. + */ +export function formatGatewayCrashLoopManualChannelStartHint(target?: { + channelId: string; + accountId?: string; +}): string { + const params = target + ? JSON.stringify({ + channel: target.channelId, + ...(target.accountId ? { accountId: target.accountId } : {}), + }) + : `{"channel":""}`; + return `Start a channel manually with: openclaw gateway call channels.start --params '${params}'`; +} const gatewayLifecycleLog = createSubsystemLogger("gateway/lifecycle"); diff --git a/src/infra/startup-migration-checkpoint.test.ts b/src/infra/startup-migration-checkpoint.test.ts index cb66115db6b5..963a35792f61 100644 --- a/src/infra/startup-migration-checkpoint.test.ts +++ b/src/infra/startup-migration-checkpoint.test.ts @@ -3,11 +3,18 @@ import { existsSync, mkdirSync } from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { closeOpenClawStateDatabaseForTest, OPENCLAW_STATE_SCHEMA_VERSION, + withOpenClawStateStartupMigrationCheckpointDatabase, } from "../state/openclaw-state-db.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; import { requireNodeSqlite } from "./node-sqlite.js"; import { acquireStartupMigrationLease, @@ -23,6 +30,32 @@ afterEach(() => { const startupMigrationTempDirs = useAutoCleanupTempDirTracker(afterEach); +type StartupMigrationLeaseTestDatabase = Pick; + +/** Rewrites only the recorded owner start time so the live owner PID looks recycled. */ +function overwriteStartupMigrationLeaseOwnerStartedAt( + env: NodeJS.ProcessEnv, + startedAt: number, +): void { + withOpenClawStateStartupMigrationCheckpointDatabase( + (db) => { + const kysely = getNodeSqliteKysely(db); + const row = executeSqliteQueryTakeFirstSync( + db, + kysely.selectFrom("state_leases").select("payload_json as payloadJson"), + ); + const payload = JSON.parse(row?.payloadJson ?? "{}") as { owner?: { startedAt?: number } }; + executeSqliteQuerySync( + db, + kysely.updateTable("state_leases").set({ + payload_json: JSON.stringify({ ...payload, owner: { ...payload.owner, startedAt } }), + }), + ); + }, + { env }, + ); +} + describe("startup migration checkpoint", () => { it("checks migration activity without creating shared state", () => { const env = { @@ -112,7 +145,7 @@ describe("startup migration checkpoint", () => { expect(hasActiveStartupMigrationLease({ env, nowMs: 1001 })).toBe(true); expect(() => acquireStartupMigrationLease({ env, nowMs: 1001, owner: "second" })).toThrow( - "OpenClaw startup migrations are already running", + `OpenClaw startup migrations are already running for this state directory; retry after the other gateway finishes or after 1970-01-01T00:05:01.000Z. (held by pid ${process.pid})`, ); lease.release(); @@ -123,6 +156,48 @@ describe("startup migration checkpoint", () => { next.release(); }); + it("reclaims an active startup migration lease whose owner process is gone", () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + const deadPid = 2_147_483_647; + const stale = acquireStartupMigrationLease({ + env, + nowMs: 1000, + owner: "stale", + ownerPid: deadPid, + }); + + expect(hasActiveStartupMigrationLease({ env, nowMs: 1001 })).toBe(false); + + const replacement = acquireStartupMigrationLease({ env, nowMs: 1001, owner: "replacement" }); + stale.release(); + expect(hasActiveStartupMigrationLease({ env, nowMs: 1002 })).toBe(true); + replacement.release(); + }); + + // PID numbers are recycled by the OS. Without the start-time guard a stale lease whose PID was + // reassigned to an unrelated live process would block startup for the full TTL. + it.skipIf(process.platform === "win32")( + "reclaims a startup migration lease whose owner PID was recycled", + () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + const stale = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "stale" }); + + // The owner PID is this live test process; only the recorded start identity is stale. + overwriteStartupMigrationLeaseOwnerStartedAt(env, 1); + + expect(hasActiveStartupMigrationLease({ env, nowMs: 1001 })).toBe(false); + + const replacement = acquireStartupMigrationLease({ env, nowMs: 1001, owner: "replacement" }); + stale.release(); + expect(hasActiveStartupMigrationLease({ env, nowMs: 1002 })).toBe(true); + replacement.release(); + }, + ); + it("does not report an expired startup migration lease as active", () => { const env = { OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), diff --git a/src/infra/startup-migration-checkpoint.ts b/src/infra/startup-migration-checkpoint.ts index 7f002f8ea12f..d8ba382f7861 100644 --- a/src/infra/startup-migration-checkpoint.ts +++ b/src/infra/startup-migration-checkpoint.ts @@ -2,7 +2,10 @@ import { randomUUID } from "node:crypto"; import { existsSync } from "node:fs"; import { createRequire } from "node:module"; +import { hostname } from "node:os"; import type { DatabaseSync } from "node:sqlite"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { getFileLockProcessStartTime, isPidDefinitelyDead } from "../shared/pid-alive.js"; import { withOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { withOpenClawStateStartupMigrationCheckpointDatabase } from "../state/openclaw-state-db.js"; @@ -32,6 +35,60 @@ export type StartupMigrationLease = { readonly owner: string; }; +type StartupMigrationLeaseOwner = { + pid: number; + host: string; + startedAt: number | null; +}; + +function parseStartupMigrationLeaseOwner( + payloadJson: string | null, +): StartupMigrationLeaseOwner | null { + if (!payloadJson) { + return null; + } + let owner: unknown; + try { + const parsed: unknown = JSON.parse(payloadJson); + owner = isRecord(parsed) ? parsed.owner : null; + } catch { + return null; + } + if (!isRecord(owner)) { + return null; + } + const { pid, host, startedAt } = owner; + if ( + typeof pid !== "number" || + !Number.isSafeInteger(pid) || + pid <= 0 || + typeof host !== "string" || + !host || + (startedAt !== null && + (typeof startedAt !== "number" || !Number.isSafeInteger(startedAt) || startedAt < 0)) + ) { + return null; + } + return { pid, host, startedAt }; +} + +function isStartupMigrationLeaseOwnerDefinitelyGone( + owner: StartupMigrationLeaseOwner | null, +): boolean { + // Reclaim only same-host owners whose PID identity is provably gone. + // The recorded start time prevents PID reuse from making a stale lease look live. + if (!owner || owner.host !== hostname()) { + return false; + } + if (isPidDefinitelyDead(owner.pid)) { + return true; + } + const currentStartedAt = getFileLockProcessStartTime(owner.pid); + return ( + owner.startedAt !== null && currentStartedAt !== null && currentStartedAt !== owner.startedAt + ); +} + function formatStartupMigrationCheckpoint(version: string, buildIdentity: string): string { return `${version}${STARTUP_MIGRATION_BUILD_SEPARATOR}${buildIdentity}`; } @@ -111,15 +168,19 @@ export function hasActiveStartupMigrationLease( return withOpenClawStateDatabaseReadOnly( ({ db }) => { const stateDb = getNodeSqliteKysely(db); + const lease = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("state_leases") + .select("payload_json as payloadJson") + .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) + .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY) + .where("expires_at", ">", nowMs), + ); return Boolean( - executeSqliteQueryTakeFirstSync( - db, - stateDb - .selectFrom("state_leases") - .select("owner") - .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) - .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY) - .where("expires_at", ">", nowMs), + lease && + !isStartupMigrationLeaseOwnerDefinitelyGone( + parseStartupMigrationLeaseOwner(lease.payloadJson), ), ); }, @@ -153,11 +214,19 @@ export function acquireStartupMigrationLease( env?: NodeJS.ProcessEnv; nowMs?: number; owner?: string; + /** Process id that owns the startup migration work. */ + ownerPid?: number; } = {}, ): StartupMigrationLease { const env = params.env ?? process.env; const nowMs = params.nowMs ?? Date.now(); const owner = params.owner ?? randomUUID(); + const ownerPid = params.ownerPid ?? process.pid; + const leaseOwner: StartupMigrationLeaseOwner = { + pid: ownerPid, + host: hostname(), + startedAt: getFileLockProcessStartTime(ownerPid), + }; const expiresAt = nowMs + STARTUP_MIGRATION_LEASE_TTL_MS; writeStartupMigrationCheckpointDatabase(env, (db) => { @@ -174,13 +243,24 @@ export function acquireStartupMigrationLease( db, stateDb .selectFrom("state_leases") - .select(["owner", "expires_at as expiresAt"]) + .select(["owner", "expires_at as expiresAt", "payload_json as payloadJson"]) .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY), ); - if (existing) { + const existingOwner = parseStartupMigrationLeaseOwner(existing?.payloadJson ?? null); + if (existing && isStartupMigrationLeaseOwnerDefinitelyGone(existingOwner)) { + executeSqliteQuerySync( + db, + stateDb + .deleteFrom("state_leases") + .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) + .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY) + .where("owner", "=", existing.owner), + ); + } else if (existing) { + const ownerHint = existingOwner ? ` (held by pid ${existingOwner.pid})` : ""; throw new Error( - `OpenClaw startup migrations are already running for this state directory; retry after the other gateway finishes or after ${new Date(existing.expiresAt ?? expiresAt).toISOString()}.`, + `OpenClaw startup migrations are already running for this state directory; retry after the other gateway finishes or after ${new Date(existing.expiresAt ?? expiresAt).toISOString()}.${ownerHint}`, ); } executeSqliteQuerySync( @@ -191,7 +271,7 @@ export function acquireStartupMigrationLease( owner, expires_at: expiresAt, heartbeat_at: nowMs, - payload_json: JSON.stringify({ version: VERSION }), + payload_json: JSON.stringify({ version: VERSION, owner: leaseOwner }), created_at: nowMs, updated_at: nowMs, }),