diff --git a/docs/gateway/restart-recovery.md b/docs/gateway/restart-recovery.md index 35daec2a8118..ee9cf152420d 100644 --- a/docs/gateway/restart-recovery.md +++ b/docs/gateway/restart-recovery.md @@ -159,6 +159,14 @@ SQLite before the process exits. After boot the gateway posts the outcome back to the originating chat and dispatches a one-shot continuation turn so the agent picks up exactly where it left off, on the same channel and thread. +The sentinel's typed SQLite columns are authoritative for restart handling; +its `payload_json` value is a replay/debug shadow only. Runtime reads, writes, +and clears SQLite state without a file fallback. During the storage cutover, a +bounded state migration runs at startup and through Doctor to preserve a +validated `restart-sentinel.json` left by the older process after an update. +The migration verifies the typed row and removes the source file before normal +restart handling continues. + ## Safety valves and observability - **Crash-loop breaker:** 3 unclean boots within 5 minutes trip a breaker that diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md index ae0408e47cd4..41bd27105bcd 100644 --- a/docs/refactor/database-first.md +++ b/docs/refactor/database-first.md @@ -1256,8 +1256,12 @@ sessionId})`; create, branch, continue, list, and fork flows live in their - Gateway restart sentinel state now uses typed shared SQLite `gateway_restart_sentinel` rows instead of `restart-sentinel.json`; runtime reads sentinel kind, status, routing, message, continuation, and stats from - typed columns. `payload_json` is only a replay/debug copy. Runtime code clears - the SQLite row directly and no longer carries file cleanup plumbing. + typed columns. Those columns are authoritative; `payload_json` is only a + replay/debug shadow. Runtime read, write, and clear paths are SQLite-only. + One bounded state-migration module runs during startup and Doctor to import a + validated older post-update sentinel before normal restart recovery, verify + the typed row, and remove the source file. No steady-state runtime module + reads, writes, or cleans up the legacy file. - Gateway restart intent and supervisor handoff state now use typed shared SQLite `gateway_restart_intent` and `gateway_restart_handoff` rows instead of `gateway-restart-intent.json` and diff --git a/scripts/check-database-first-legacy-stores.mjs b/scripts/check-database-first-legacy-stores.mjs index b0d31720a59e..865e97bcf25d 100644 --- a/scripts/check-database-first-legacy-stores.mjs +++ b/scripts/check-database-first-legacy-stores.mjs @@ -84,6 +84,26 @@ const fsSafePackageModulePattern = /^@openclaw\/fs-safe(?:\/(?:root|store))?$/u; const bridgeMarkerPattern = /\btranscriptLocator\b|sqlite-transcript:\/\//u; +// The restart handoff must survive its one cutover migration without leaving +// filesystem fallback imports in the steady-state runtime owner. +const legacyRestartSentinelMigrationPath = "src/infra/state-migrations.restart-sentinel.ts"; +const legacyRestartSentinelPreflightPath = "src/cli/program/config-guard.ts"; +const legacyRestartSentinelRuntimePath = "src/infra/restart-sentinel.ts"; +const legacyRestartSentinelPreflightFilenames = new Set([ + "restart-sentinel.json", + "restart-sentinel.json.doctor-importing", +]); +const legacyRestartSentinelFilenamePattern = + /(?:^|[/\\])restart-sentinel\.json(?:\.doctor-importing)?$/u; +const legacyRestartSentinelRuntimeImportSpecifiers = new Set([ + "fs", + "fs/promises", + "node:fs", + "node:fs/promises", + "node:path", + "path", +]); + const legacyStorePatterns = [ /\bsessions\.json\b/u, /\.trajectory\.jsonl\b/u, @@ -129,6 +149,7 @@ const allowedRuntimeMigrationPaths = [ "src/infra/state-migrations.managed-outgoing-images.ts", "src/infra/state-migrations.apns.ts", "src/infra/state-migrations.mcp-oauth.ts", + legacyRestartSentinelMigrationPath, "src/infra/state-migrations.workspace-setup.ts", "src/infra/state-migrations.web-push.ts", "src/infra/state-migrations.node-host.ts", @@ -327,6 +348,86 @@ function importSource(node) { return ts.isStringLiteral(moduleSpecifier) ? moduleSpecifier.text : ""; } +function isLegacyRestartSentinelPreflightDetection(node, relativePath) { + if ( + relativePath !== legacyRestartSentinelPreflightPath || + !legacyRestartSentinelPreflightFilenames.has(node.text) + ) { + return false; + } + const joinCall = node.parent; + if ( + !ts.isCallExpression(joinCall) || + joinCall.arguments.length !== 2 || + joinCall.arguments[1] !== node || + !ts.isPropertyAccessExpression(joinCall.expression) || + !ts.isIdentifier(joinCall.expression.expression) || + joinCall.expression.expression.text !== "path" || + joinCall.expression.name.text !== "join" || + !ts.isIdentifier(joinCall.arguments[0]) || + joinCall.arguments[0].text !== "stateDir" + ) { + return false; + } + const paths = joinCall.parent; + if (!ts.isArrayLiteralExpression(paths)) { + return false; + } + const someAccess = paths.parent; + if ( + !ts.isPropertyAccessExpression(someAccess) || + someAccess.expression !== paths || + someAccess.name.text !== "some" + ) { + return false; + } + const someCall = someAccess.parent; + return ( + ts.isCallExpression(someCall) && + someCall.arguments.length === 1 && + ts.isIdentifier(someCall.arguments[0]) && + someCall.arguments[0].text === "fileOrDirExists" + ); +} + +function collectLegacyRestartSentinelBoundaryViolations(sourceFile, relativePath) { + if (relativePath === legacyRestartSentinelMigrationPath) { + return []; + } + + const violations = []; + const seen = new Set(); + function add(node, kind) { + const line = toLine(sourceFile, node); + const key = `${line}:${kind}`; + if (seen.has(key)) { + return; + } + seen.add(key); + violations.push({ kind, line }); + } + + function visit(node) { + if ( + ts.isStringLiteralLike(node) && + legacyRestartSentinelFilenamePattern.test(node.text) && + !isLegacyRestartSentinelPreflightDetection(node, relativePath) + ) { + add(node, "legacy restart sentinel reference"); + } + if ( + relativePath === legacyRestartSentinelRuntimePath && + ts.isImportDeclaration(node) && + legacyRestartSentinelRuntimeImportSpecifiers.has(importSource(node)) + ) { + add(node, "legacy restart sentinel filesystem import"); + } + ts.forEachChild(node, visit); + } + visit(sourceFile); + return violations; +} + function isHelperWriteModuleSource(source) { return ( source === "openclaw/plugin-sdk/file-access-runtime" || @@ -526,21 +627,28 @@ function legacyCandidateTexts(sourceFile, node) { */ export function collectDatabaseFirstLegacyStoreViolations( content, - relativePath = "source.ts", + inputRelativePath = "source.ts", scanOptions = {}, ) { + const relativePath = inputRelativePath.replaceAll("\\", "/"); + const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true); + const boundaryViolations = collectLegacyRestartSentinelBoundaryViolations( + sourceFile, + relativePath, + ); if (isAllowedLegacyOwnerPath(relativePath)) { - return []; + return boundaryViolations; } - const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true); const currentLegacyWriteAllowances = scanOptions.currentLegacyWriteAllowances ?? currentLegacyWriteViolationAllowances(relativePath); const createRequireBindings = collectCreateRequireBindings(sourceFile); const { fsModuleBindings, fsWriteAliases, fsSafeStoreFactoryAliases } = collectFsBindings(sourceFile); - const violations = []; - const seenViolations = new Set(); + const violations = [...boundaryViolations]; + const seenViolations = new Set( + boundaryViolations.map((violation) => `${violation.line}:${violation.kind}`), + ); const fsModuleBindingScopes = [new Map([...fsModuleBindings].map((name) => [name, true]))]; const fsModulePropertyScopes = [new Map()]; const fsWriteAliasScopes = [fsWriteAliases]; diff --git a/src/cli/program/config-guard.test.ts b/src/cli/program/config-guard.test.ts index 90b2bb725dbd..57de193caeba 100644 --- a/src/cli/program/config-guard.test.ts +++ b/src/cli/program/config-guard.test.ts @@ -207,6 +207,23 @@ describe("ensureConfigReady", () => { }); }); + it.each(["restart-sentinel.json", "restart-sentinel.json.doctor-importing"])( + "runs doctor flow when lightweight startup detection finds %s", + async (relativePath) => { + const root = useTempOpenClawHome(); + writeStateMarker(root, relativePath); + + await runEnsureConfigReady(["status"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({ + migrateState: true, + migrateLegacyConfig: false, + invalidConfigNote: false, + observe: false, + }); + }, + ); + it("runs doctor flow when lightweight startup detection finds a pending SQLite archive", async () => { const root = useTempOpenClawHome(); writePendingTaskSidecarArchiveMarker(root); diff --git a/src/cli/program/config-guard.ts b/src/cli/program/config-guard.ts index 8d0d8f02018e..8978a55a29c6 100644 --- a/src/cli/program/config-guard.ts +++ b/src/cli/program/config-guard.ts @@ -124,6 +124,8 @@ function hasLegacyStateMigrationInputs(): boolean { path.join(stateDir, "agent"), path.join(stateDir, "agents"), path.join(stateDir, "plugins", "installs.json"), + path.join(stateDir, "restart-sentinel.json"), + path.join(stateDir, "restart-sentinel.json.doctor-importing"), path.join(stateDir, "sessions"), path.join(stateDir, "state", "openclaw.sqlite"), ].some(fileOrDirExists) || diff --git a/src/gateway/server-restart-sentinel-notice.test.ts b/src/gateway/server-restart-sentinel-notice.test.ts new file mode 100644 index 000000000000..1c63c2304856 --- /dev/null +++ b/src/gateway/server-restart-sentinel-notice.test.ts @@ -0,0 +1,173 @@ +// Exercises restart-notice retries against the real SQLite outbound queue. +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { getDeliveryQueueEntryStatus } from "../infra/delivery-queue-sqlite.js"; +import { PlatformMessageNotDispatchedError } from "../infra/outbound/deliver-types.js"; +import { loadPendingDelivery } from "../infra/outbound/delivery-queue-storage.js"; +import { markDeliveryPlatformSendAttemptStarted } from "../infra/outbound/delivery-queue.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; + +const mocks = vi.hoisted(() => ({ + sendDurableMessageBatch: vi.fn(), + recoveryDeliver: vi.fn(), + resolveOutboundChannelMessageAdapter: vi.fn(() => undefined), + sleep: vi.fn(async () => {}), +})); + +vi.mock("../channels/message/runtime.js", () => ({ + sendDurableMessageBatch: mocks.sendDurableMessageBatch, +})); + +vi.mock("../infra/outbound/deliver.js", () => ({ + deliverOutboundPayloadsInternal: mocks.recoveryDeliver, +})); + +vi.mock("../infra/outbound/channel-resolution.js", () => ({ + resolveOutboundChannelMessageAdapter: mocks.resolveOutboundChannelMessageAdapter, +})); + +vi.mock("../utils/sleep.js", () => ({ sleep: mocks.sleep })); + +const { deliverRestartSentinelNotice, enqueueRestartSentinelNotice } = + await import("./server-restart-sentinel-notice.js"); + +type DeliveryRequest = { deliveryQueueId?: string; deliveryQueueStateDir?: string }; + +describe("restart sentinel notice recovery", () => { + let envSnapshot: ReturnType | undefined; + let stateDir = ""; + const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { + afterEach(() => { + closeOpenClawStateDatabaseForTest(); + envSnapshot?.restore(); + envSnapshot = undefined; + cleanup(); + }); + }); + + beforeEach(() => { + closeOpenClawStateDatabaseForTest(); + stateDir = tempDirs.make("openclaw-restart-notice-"); + envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + mocks.sendDurableMessageBatch.mockReset(); + mocks.recoveryDeliver.mockReset(); + mocks.resolveOutboundChannelMessageAdapter.mockClear(); + mocks.sleep.mockClear(); + }); + + async function enqueueNotice(): Promise { + const queued = await enqueueRestartSentinelNotice({ + channel: "whatsapp", + to: "+15550002", + message: "restart complete", + sessionKey: "agent:main:main", + revision: 123, + }); + return queued.id; + } + + async function deliverNotice(queueId: string): Promise { + await deliverRestartSentinelNotice({ + deps: {} as never, + cfg: {}, + channel: "whatsapp", + to: "+15550002", + message: "restart complete", + sessionKey: "agent:main:main", + summary: "restart summary", + queueId, + }); + } + + async function markAttempt(request: unknown): Promise { + const { deliveryQueueId, deliveryQueueStateDir } = request as DeliveryRequest; + if (!deliveryQueueId) { + throw new Error("expected durable delivery queue id"); + } + await markDeliveryPlatformSendAttemptStarted(deliveryQueueId, deliveryQueueStateDir); + } + + function queueStatus(queueId: string): string | undefined { + return getDeliveryQueueEntryStatus("outbound", queueId, stateDir); + } + + it("replays a retryable provider-not-dispatched failure after the startup scan", async () => { + const queueId = await enqueueNotice(); + mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => { + await markAttempt(request); + return { + status: "failed", + error: new PlatformMessageNotDispatchedError("connect failed before dispatch", { + cause: new Error("connect failed"), + }), + }; + }); + mocks.recoveryDeliver.mockResolvedValueOnce([ + { channel: "whatsapp", messageId: "recovered-1" }, + ]); + + await deliverNotice(queueId); + + expect(mocks.recoveryDeliver).toHaveBeenCalledOnce(); + expect(await loadPendingDelivery(queueId)).toBeNull(); + expect(queueStatus(queueId)).toBe("completed"); + }); + + it("does not blindly resend an ambiguous platform attempt", async () => { + const queueId = await enqueueNotice(); + mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => { + await markAttempt(request); + return { status: "failed", error: new Error("platform outcome unknown") }; + }); + + await deliverNotice(queueId); + + expect(mocks.recoveryDeliver).not.toHaveBeenCalled(); + expect(await loadPendingDelivery(queueId)).toBeNull(); + expect(queueStatus(queueId)).toBe("failed"); + }); + + it("dead-letters a permanent provider rejection without replay", async () => { + const queueId = await enqueueNotice(); + mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => { + await markAttempt(request); + return { + status: "failed", + error: new PlatformMessageNotDispatchedError("payload rejected", { + cause: new Error("invalid payload"), + retryable: false, + }), + }; + }); + + await deliverNotice(queueId); + + expect(mocks.recoveryDeliver).not.toHaveBeenCalled(); + expect(await loadPendingDelivery(queueId)).toBeNull(); + expect(queueStatus(queueId)).toBe("failed"); + }); + + it("preserves the shipped 45-attempt budget before dead-lettering", async () => { + const queueId = await enqueueNotice(); + const retryableFailure = () => + new PlatformMessageNotDispatchedError("transport unavailable before dispatch", { + cause: new Error("transport unavailable"), + }); + mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => { + await markAttempt(request); + return { status: "failed", error: retryableFailure() }; + }); + mocks.recoveryDeliver.mockImplementation(async (request) => { + await markAttempt(request); + throw retryableFailure(); + }); + + await deliverNotice(queueId); + + expect(mocks.sendDurableMessageBatch).toHaveBeenCalledOnce(); + expect(mocks.recoveryDeliver).toHaveBeenCalledTimes(44); + expect(queueStatus(queueId)).toBe("failed"); + }); +}); diff --git a/src/gateway/server-restart-sentinel-notice.ts b/src/gateway/server-restart-sentinel-notice.ts new file mode 100644 index 000000000000..9458b7b929e8 --- /dev/null +++ b/src/gateway/server-restart-sentinel-notice.ts @@ -0,0 +1,260 @@ +// Durable outbound notice ownership for restart-sentinel recovery. +import { sendDurableMessageBatch } from "../channels/message/runtime.js"; +import type { CliDeps } from "../cli/deps.types.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { + findPlatformMessageRejectedError, + isProvenDeliveryNotSentError, +} from "../infra/delivery-recovery.shared.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import { deliverOutboundPayloadsInternal } from "../infra/outbound/deliver.js"; +import { + failPendingDelivery, + loadPendingDelivery, + reserveDeliveryAttempt, +} from "../infra/outbound/delivery-queue-storage.js"; +import { + ackDelivery, + drainPendingDeliveries, + enqueueDeliveryOnce, + failDelivery, + failDeliveryAfterPlatformSend, + failDeliveryBeforePlatformSend, + withActiveDeliveryClaim, +} from "../infra/outbound/delivery-queue.js"; +import { buildOutboundSessionContext } from "../infra/outbound/session-context.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; + +const log = createSubsystemLogger("gateway/restart-sentinel"); +const RESTART_NOTICE_RECOVERY_DELAY_MS = process.env.VITEST ? 1 : 1_000; +const RESTART_NOTICE_MAX_ATTEMPTS = 45; +const RESTART_NOTICE_RECOVERY_MAX_CYCLES = RESTART_NOTICE_MAX_ATTEMPTS + 1; + +type RestartSentinelNoticeRoute = { + channel: string; + to: string; + accountId?: string; + replyToId?: string; + threadId?: string; +}; + +export async function enqueueRestartSentinelNotice( + params: RestartSentinelNoticeRoute & { + message: string; + sessionKey: string; + revision: number; + }, +): Promise<{ id: string; created: boolean }> { + return await enqueueDeliveryOnce( + { + channel: params.channel, + to: params.to, + accountId: params.accountId, + replyToId: params.replyToId, + threadId: params.threadId, + payloads: [{ text: params.message }], + bestEffort: false, + completionRetention: "permanent", + maxRetries: RESTART_NOTICE_MAX_ATTEMPTS, + }, + `restart-sentinel-notice:${params.sessionKey}:${params.revision}`, + ); +} + +async function waitForRecoveryDrain(): Promise { + await new Promise((resolve) => { + const timer = setTimeout(resolve, RESTART_NOTICE_RECOVERY_DELAY_MS); + timer.unref?.(); + }); +} + +async function drainFailedRestartSentinelNotice(params: { + cfg: OpenClawConfig; + queueId: string; + sessionKey: string; + summary: string; +}): Promise { + for (let cycle = 1; cycle <= RESTART_NOTICE_RECOVERY_MAX_CYCLES; cycle += 1) { + const beforeDrain = await loadPendingDelivery(params.queueId).catch((error: unknown) => { + log.warn(`${params.summary}: restart notice recovery reload failed: ${String(error)}`, { + queueId: params.queueId, + sessionKey: params.sessionKey, + cycle, + }); + return undefined; + }); + if (beforeDrain === null) { + return; + } + const attemptCount = beforeDrain + ? Math.max(beforeDrain.attemptCount ?? 0, beforeDrain.retryCount) + : 0; + // Atomic queue reservation blocks attempt 46. Exhausted rows get an + // immediate terminal drain; live retry attempts retain one-second spacing. + if (attemptCount < RESTART_NOTICE_MAX_ATTEMPTS) { + await waitForRecoveryDrain(); + } + await drainPendingDeliveries({ + drainKey: `restart-recovery:${params.queueId}`, + logLabel: `${params.summary}: restart notice recovery`, + cfg: params.cfg, + log, + deliver: deliverOutboundPayloadsInternal, + selectEntry: (entry) => ({ + match: entry.id === params.queueId, + // The caller already waits between attempts. Recovery still reconciles + // send-attempt evidence before it permits recipient-visible replay. + bypassBackoff: true, + }), + }).catch((error: unknown) => { + log.warn(`${params.summary}: restart notice recovery drain failed: ${String(error)}`, { + queueId: params.queueId, + sessionKey: params.sessionKey, + cycle, + }); + }); + } + const pending = await loadPendingDelivery(params.queueId).catch((error: unknown) => { + log.warn(`${params.summary}: restart notice terminal reload failed: ${String(error)}`, { + queueId: params.queueId, + sessionKey: params.sessionKey, + }); + return undefined; + }); + if (pending === null) { + return; + } + log.warn(`${params.summary}: restart notice remains queued after bounded recovery`, { + queueId: params.queueId, + sessionKey: params.sessionKey, + retryCount: pending?.retryCount ?? null, + attemptCount: pending?.attemptCount ?? null, + maxAttempts: RESTART_NOTICE_MAX_ATTEMPTS, + }); +} + +export async function deliverRestartSentinelNotice( + params: RestartSentinelNoticeRoute & { + deps: CliDeps; + cfg: OpenClawConfig; + sessionKey: string; + summary: string; + message: string; + queueId: string; + }, +): Promise { + const claim = await withActiveDeliveryClaim(params.queueId, async () => { + try { + const reservation = await reserveDeliveryAttempt(params.queueId, RESTART_NOTICE_MAX_ATTEMPTS); + if (reservation.status === "exhausted") { + return false; + } + } catch (err) { + log.warn( + `${params.summary}: outbound delivery attempt reservation failed; queued for recovery: ${formatErrorMessage(err)}`, + { + channel: params.channel, + to: params.to, + sessionKey: params.sessionKey, + }, + ); + return false; + } + try { + const send = await sendDurableMessageBatch({ + cfg: params.cfg, + channel: params.channel, + to: params.to, + accountId: params.accountId, + replyToId: params.replyToId, + threadId: params.threadId, + payloads: [{ text: params.message }], + session: buildOutboundSessionContext({ cfg: params.cfg, sessionKey: params.sessionKey }), + deps: params.deps, + bestEffort: false, + skipQueue: true, + deliveryQueueId: params.queueId, + }); + if (send.status === "failed" || send.status === "partial_failed") { + throw send.error; + } + const results = send.status === "sent" ? send.results : []; + if (results.length === 0) { + throw new Error("outbound delivery returned no results"); + } + try { + await ackDelivery(params.queueId); + return true; + } catch (err) { + const error = formatErrorMessage(err); + await failDeliveryAfterPlatformSend(params.queueId, error).catch(() => undefined); + log.warn(`${params.summary}: outbound delivery ack failed; queued for recovery: ${error}`, { + channel: params.channel, + to: params.to, + sessionKey: params.sessionKey, + }); + return false; + } + } catch (err) { + // The send path records platform-attempt evidence on this queue row. + // Durable recovery owns retries so ambiguous outcomes are reconciled + // before another recipient-visible send can begin. + const error = formatErrorMessage(err); + const permanentRejection = findPlatformMessageRejectedError(err); + if (permanentRejection) { + try { + const pending = await loadPendingDelivery(params.queueId); + if (pending) { + await failPendingDelivery({ + id: params.queueId, + expectedStatus: "pending", + lastError: error, + entry: pending, + }); + } + } catch (persistError) { + log.warn( + `${params.summary}: permanent rejection persistence failed; queued for recovery: ${formatErrorMessage(persistError)}`, + { + channel: params.channel, + to: params.to, + sessionKey: params.sessionKey, + }, + ); + return false; + } + log.warn(`${params.summary}: outbound delivery permanently rejected: ${error}`, { + channel: params.channel, + to: params.to, + sessionKey: params.sessionKey, + }); + return true; + } + const recordFailure = isProvenDeliveryNotSentError(err) + ? failDeliveryBeforePlatformSend + : failDelivery; + await recordFailure(params.queueId, error).catch(() => undefined); + log.warn(`${params.summary}: outbound delivery failed; queued for recovery: ${String(err)}`, { + channel: params.channel, + to: params.to, + sessionKey: params.sessionKey, + }); + return false; + } + }); + if (claim.status === "claimed-by-other-owner") { + log.info(`${params.summary}: durable restart notice claimed by recovery`, { + sessionKey: params.sessionKey, + }); + } + const needsRecovery = + claim.status === "claimed-by-other-owner" || (claim.status === "claimed" && !claim.value); + if (needsRecovery) { + await drainFailedRestartSentinelNotice({ + cfg: params.cfg, + queueId: params.queueId, + sessionKey: params.sessionKey, + summary: params.summary, + }); + } +} diff --git a/src/gateway/server-restart-sentinel.test.ts b/src/gateway/server-restart-sentinel.test.ts index 9c8ede22a01d..08c29d620380 100644 --- a/src/gateway/server-restart-sentinel.test.ts +++ b/src/gateway/server-restart-sentinel.test.ts @@ -25,16 +25,22 @@ type AdvanceSessionDeliveryAgentRunMock = const mocks = vi.hoisted(() => { const state = { - queuedSessionDelivery: null as Record | null, + queuedSessionDeliveries: new Map>(), + nextSessionDeliveryId: 1, }; return { resolveSessionAgentId: vi.fn(() => "agent-from-key"), get queuedSessionDelivery() { - return state.queuedSessionDelivery; + return state.queuedSessionDeliveries.values().next().value ?? null; }, set queuedSessionDelivery(value: Record | null) { - state.queuedSessionDelivery = value; + state.queuedSessionDeliveries.clear(); + state.nextSessionDeliveryId = 1; + if (value) { + state.queuedSessionDeliveries.set("session-delivery-1", value); + state.nextSessionDeliveryId = 2; + } }, dispatchGatewayMethodInProcess: vi.fn(async () => ({ status: "ok", @@ -46,6 +52,7 @@ const mocks = vi.hoisted(() => { readRestartSentinel: vi.fn( async (): Promise => ({ version: 1, + revision: 123, payload: { kind: "restart", status: "ok", @@ -60,7 +67,7 @@ const mocks = vi.hoisted(() => { }), ), finalizeUpdateRestartSentinelRunningVersion: vi.fn(async () => null), - clearRestartSentinel: vi.fn(async () => undefined), + clearRestartSentinelIfRevision: vi.fn(async () => true), formatRestartSentinelMessage: vi.fn(() => "restart message"), summarizeRestartSentinel: vi.fn(() => "restart summary"), resolveMainSessionKeyFromConfig: vi.fn(() => "agent:main:main"), @@ -100,14 +107,34 @@ const mocks = vi.hoisted(() => { to: "+15550002", })) as (params?: { to?: string }) => { ok: true; to: string } | { ok: false; error: Error }), deliverOutboundPayloads: vi.fn(async () => [{ channel: "whatsapp", messageId: "msg-1" }]), - enqueueDelivery: vi.fn(async () => "queue-1"), + enqueueDeliveryOnce: vi.fn(async (_payload: unknown, id: string) => ({ id, created: true })), ackDelivery: vi.fn(async () => {}), failDelivery: vi.fn(async () => {}), + failDeliveryAfterPlatformSend: vi.fn(async () => {}), + failDeliveryBeforePlatformSend: vi.fn(async () => {}), + failPendingDelivery: vi.fn(async () => ({ status: "failed" as const })), + loadPendingDelivery: vi.fn(async () => null), + drainPendingDeliveries: vi.fn(async () => {}), + reserveDeliveryAttempt: vi.fn(async () => ({ + status: "reserved" as const, + attemptCount: 1, + })), + withActiveDeliveryClaim: vi.fn(async (_id: string, fn: () => Promise) => ({ + status: "claimed" as const, + value: await fn(), + })), enqueueSystemEvent: vi.fn(), requestHeartbeat: vi.fn(), enqueueSessionDelivery: vi.fn(async (payload: Record) => { - state.queuedSessionDelivery = payload; - return "session-delivery-1"; + const existing = [...state.queuedSessionDeliveries.entries()].find( + ([, entry]) => entry.idempotencyKey === payload.idempotencyKey, + ); + if (existing) { + return existing[0]; + } + const id = `session-delivery-${state.nextSessionDeliveryId++}`; + state.queuedSessionDeliveries.set(id, payload); + return id; }), ackSessionDelivery: vi.fn(async () => {}), advanceSessionDeliveryAgentRun: vi.fn(async () => {}), @@ -122,7 +149,9 @@ const mocks = vi.hoisted(() => { messageId: "generated-media-transcript", })), removeCronRunContinuationSessionIfIdle: vi.fn(async () => {}), - loadPendingSessionDelivery: vi.fn(async () => state.queuedSessionDelivery), + loadPendingSessionDelivery: vi.fn( + async (id: string) => state.queuedSessionDeliveries.get(id) ?? null, + ), drainPendingSessionDeliveries: vi.fn( async (params: { logLabel: string; @@ -130,7 +159,14 @@ const mocks = vi.hoisted(() => { selectEntry: (entry: Record, now: number) => { match: boolean }; deliver: (entry: Record) => Promise; }) => { - if (!state.queuedSessionDelivery) { + const selected = [...state.queuedSessionDeliveries.entries()] + .map(([id, payload]) => ({ id, payload })) + .find( + ({ id, payload }) => + params.selectEntry({ id, enqueuedAt: 1, retryCount: 0, ...payload }, Date.now()) + .match, + ); + if (!selected) { return; } const entry: Record & { @@ -138,18 +174,14 @@ const mocks = vi.hoisted(() => { enqueuedAt: number; retryCount: number; } = { - id: "session-delivery-1", + id: selected.id, enqueuedAt: 1, retryCount: 0, - ...state.queuedSessionDelivery, + ...selected.payload, }; - const decision = params.selectEntry(entry, Date.now()); - if (!decision.match) { - return; - } const maxRetries = typeof entry["maxRetries"] === "number" ? entry["maxRetries"] : 5; if (entry.retryCount >= maxRetries) { - state.queuedSessionDelivery = null; + state.queuedSessionDeliveries.delete(entry.id); params.log.warn( `${params.logLabel}: entry ${entry.id} exceeded max retries and was moved to failed/`, ); @@ -157,13 +189,13 @@ const mocks = vi.hoisted(() => { } try { await params.deliver(entry); - state.queuedSessionDelivery = null; + state.queuedSessionDeliveries.delete(entry.id); } catch (err) { - state.queuedSessionDelivery = { + state.queuedSessionDeliveries.set(entry.id, { ...entry, retryCount: entry.retryCount + 1, lastError: err instanceof Error ? err.message : String(err), - }; + }); params.log.warn(`${params.logLabel}: retry failed for entry ${entry.id}: ${String(err)}`); } }, @@ -215,7 +247,7 @@ vi.mock("../agents/agent-scope.js", async () => { vi.mock("../infra/restart-sentinel.js", () => ({ finalizeUpdateRestartSentinelRunningVersion: mocks.finalizeUpdateRestartSentinelRunningVersion, readRestartSentinel: mocks.readRestartSentinel, - clearRestartSentinel: mocks.clearRestartSentinel, + clearRestartSentinelIfRevision: mocks.clearRestartSentinelIfRevision, formatRestartSentinelMessage: mocks.formatRestartSentinelMessage, summarizeRestartSentinel: mocks.summarizeRestartSentinel, })); @@ -321,9 +353,19 @@ vi.mock("../infra/outbound/deliver.js", () => ({ })); vi.mock("../infra/outbound/delivery-queue.js", () => ({ - enqueueDelivery: mocks.enqueueDelivery, + enqueueDeliveryOnce: mocks.enqueueDeliveryOnce, ackDelivery: mocks.ackDelivery, failDelivery: mocks.failDelivery, + failDeliveryAfterPlatformSend: mocks.failDeliveryAfterPlatformSend, + failDeliveryBeforePlatformSend: mocks.failDeliveryBeforePlatformSend, + drainPendingDeliveries: mocks.drainPendingDeliveries, + withActiveDeliveryClaim: mocks.withActiveDeliveryClaim, +})); + +vi.mock("../infra/outbound/delivery-queue-storage.js", () => ({ + failPendingDelivery: mocks.failPendingDelivery, + loadPendingDelivery: mocks.loadPendingDelivery, + reserveDeliveryAttempt: mocks.reserveDeliveryAttempt, })); vi.mock("../infra/system-events.js", () => ({ @@ -452,6 +494,7 @@ describe("scheduleRestartSentinelWake", () => { mocks.readRestartSentinel.mockReset(); mocks.readRestartSentinel.mockResolvedValue({ version: 1, + revision: 123, payload: { kind: "restart", status: "ok", @@ -488,10 +531,18 @@ describe("scheduleRestartSentinelWake", () => { mocks.resolveOutboundTarget.mockReturnValue({ ok: true as const, to: "+15550002" }); mocks.deliverOutboundPayloads.mockReset(); mocks.deliverOutboundPayloads.mockResolvedValue([{ channel: "whatsapp", messageId: "msg-1" }]); - mocks.enqueueDelivery.mockReset(); - mocks.enqueueDelivery.mockResolvedValue("queue-1"); + mocks.enqueueDeliveryOnce.mockReset(); + mocks.enqueueDeliveryOnce.mockImplementation(async (_payload, id) => ({ id, created: true })); mocks.ackDelivery.mockClear(); mocks.failDelivery.mockClear(); + mocks.failDeliveryAfterPlatformSend.mockClear(); + mocks.failDeliveryBeforePlatformSend.mockClear(); + mocks.failPendingDelivery.mockClear(); + mocks.loadPendingDelivery.mockReset(); + mocks.loadPendingDelivery.mockResolvedValue(null); + mocks.drainPendingDeliveries.mockClear(); + mocks.reserveDeliveryAttempt.mockClear(); + mocks.withActiveDeliveryClaim.mockClear(); mocks.enqueueSystemEvent.mockClear(); mocks.requestHeartbeat.mockClear(); mocks.enqueueSessionDelivery.mockClear(); @@ -509,7 +560,8 @@ describe("scheduleRestartSentinelWake", () => { mocks.recoverPendingSessionDeliveries.mockClear(); mocks.finalizeUpdateRestartSentinelRunningVersion.mockReset(); mocks.finalizeUpdateRestartSentinelRunningVersion.mockResolvedValue(null); - mocks.clearRestartSentinel.mockClear(); + mocks.clearRestartSentinelIfRevision.mockReset(); + mocks.clearRestartSentinelIfRevision.mockResolvedValue(true); mocks.formatRestartSentinelMessage.mockClear(); mocks.summarizeRestartSentinel.mockClear(); mocks.injectTimestamp.mockClear(); @@ -533,14 +585,21 @@ describe("scheduleRestartSentinelWake", () => { deps, bestEffort: false, skipQueue: true, + deliveryQueueId: "restart-sentinel-notice:agent:main:main:123", }); - expectMockCallFields(mocks.enqueueDelivery, { + expectMockCallFields(mocks.enqueueDeliveryOnce, { channel: "whatsapp", to: "+15550002", payloads: [{ text: "restart message" }], bestEffort: false, + completionRetention: "permanent", + maxRetries: 45, }); - expect(mocks.ackDelivery).toHaveBeenCalledWith("queue-1"); + expect(mocks.ackDelivery).toHaveBeenCalledWith("restart-sentinel-notice:agent:main:main:123"); + expect(mocks.reserveDeliveryAttempt).toHaveBeenCalledWith( + "restart-sentinel-notice:agent:main:main:123", + 45, + ); expect(mocks.failDelivery).not.toHaveBeenCalled(); expect(mocks.formatRestartSentinelMessage).toHaveBeenCalledWith(expect.anything()); expect(mocks.summarizeRestartSentinel).toHaveBeenCalledWith(expect.anything()); @@ -558,60 +617,263 @@ describe("scheduleRestartSentinelWake", () => { expect(mocks.logWarn).not.toHaveBeenCalled(); }); - it("retries outbound delivery once and logs a warning without dropping the agent wake", async () => { - vi.useFakeTimers(); - mocks.deliverOutboundPayloads - .mockRejectedValueOnce(new Error("transport not ready")) - .mockResolvedValueOnce([{ channel: "whatsapp", messageId: "msg-2" }]); + it("persists every downstream intent before consuming the loaded revision", async () => { + await scheduleRestartSentinelWake({ deps: {} as never }); - const wakePromise = scheduleRestartSentinelWake({ deps: {} as never }); - await Promise.resolve(); - await Promise.resolve(); - await vi.advanceTimersByTimeAsync(1_000); - await wakePromise; + expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123); + const clearOrder = mocks.clearRestartSentinelIfRevision.mock.invocationCallOrder[0] ?? 0; + expect(mocks.enqueueSessionDelivery.mock.invocationCallOrder[0]).toBeLessThan(clearOrder); + expect(mocks.enqueueDeliveryOnce.mock.invocationCallOrder[0]).toBeLessThan(clearOrder); + expect(clearOrder).toBeLessThan(mocks.enqueueSystemEvent.mock.invocationCallOrder[0] ?? 0); + expect(clearOrder).toBeLessThan(mocks.deliverOutboundPayloads.mock.invocationCallOrder[0] ?? 0); + }); - expect(mocks.enqueueDelivery).toHaveBeenCalledTimes(1); - expect(mocks.deliverOutboundPayloads).toHaveBeenCalledTimes(2); - expectMockCallFields(mocks.deliverOutboundPayloads, { skipQueue: true }, 0); - expectMockCallFields(mocks.deliverOutboundPayloads, { skipQueue: true }, 1); - expect(mocks.ackDelivery).toHaveBeenCalledWith("queue-1"); + it("stops delivery when guarded sentinel consumption fails", async () => { + mocks.clearRestartSentinelIfRevision.mockRejectedValueOnce(new Error("database locked")); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.enqueueSessionDelivery).toHaveBeenCalledOnce(); + expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledOnce(); + expect(mocks.enqueueSystemEvent).not.toHaveBeenCalled(); + expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled(); + expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", { + source: "restart-sentinel", + sessionKey: "agent:main:main", + reason: "database locked", + }); + }); + + it("preserves a newer sentinel while draining durable work from the loaded revision", async () => { + mocks.clearRestartSentinelIfRevision.mockResolvedValueOnce(false); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123); + expect(mocks.enqueueSystemEvent).toHaveBeenCalledOnce(); + expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce(); + expect(mocks.logInfo).toHaveBeenCalledWith( + "restart summary: newer restart sentinel preserved while draining durable work", + { sessionKey: "agent:main:main" }, + ); + }); + + it("does not resend a restart notice whose stable queue id is already owned", async () => { + mocks.enqueueDeliveryOnce.mockImplementationOnce(async (_payload, id) => ({ + id, + created: false, + })); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123); + expect(mocks.enqueueDeliveryOnce.mock.calls[0]?.[1]).toBe( + "restart-sentinel-notice:agent:main:main:123", + ); + expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled(); + expect(mocks.ackDelivery).not.toHaveBeenCalled(); expect(mocks.failDelivery).not.toHaveBeenCalled(); - expect(mocks.enqueueSystemEvent).toHaveBeenCalledTimes(1); - expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1); - expect(mocks.logWarn.mock.calls).toEqual([ - [ - "restart summary: outbound delivery failed; retrying in 1000ms: Error: transport not ready", - { - channel: "whatsapp", - to: "+15550002", - sessionKey: "agent:main:main", - attempt: 1, - maxAttempts: 45, - }, - ], + expect(mocks.logInfo).toHaveBeenCalledWith( + "restart summary: durable restart notice already owned", + { sessionKey: "agent:main:main" }, + ); + }); + + it("queues the restart wake before a system-event continuation", async () => { + mocks.readRestartSentinel.mockResolvedValueOnce({ + version: 1, + revision: 123, + payload: { + kind: "restart", + status: "ok", + ts: 99, + sessionKey: "agent:main:main", + continuation: { kind: "systemEvent", text: "continue" }, + }, + }); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.enqueueSessionDelivery).toHaveBeenCalledTimes(2); + expect(mocks.enqueueSessionDelivery).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + text: "restart message", + idempotencyKey: "restart-sentinel-wake:agent:main:main:123", + completionRetention: "permanent", + }), + ); + expect(mocks.enqueueSessionDelivery).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + text: "continue", + idempotencyKey: "restart-sentinel:agent:main:main:systemEvent:123", + completionRetention: "permanent", + }), + ); + expect(mocks.enqueueSystemEvent.mock.calls.map((call) => call[0])).toEqual([ + "restart message", + "continue", ]); }); - it("keeps one queued restart notice when outbound retries are exhausted", async () => { - vi.useFakeTimers(); - mocks.deliverOutboundPayloads.mockRejectedValue(new Error("transport still not ready")); + it("queues a failed outbound notice for durable recovery without dropping the agent wake", async () => { + mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("platform outcome unknown")); + mocks.loadPendingDelivery + .mockResolvedValueOnce({ + id: "restart-sentinel-notice:agent:main:main:123", + retryCount: 1, + lastError: "platform outcome unknown", + } as never) + .mockResolvedValue(null); - const wakePromise = scheduleRestartSentinelWake({ deps: {} as never }); - await Promise.resolve(); - await Promise.resolve(); - await vi.advanceTimersByTimeAsync(44_000); - await wakePromise; + await scheduleRestartSentinelWake({ deps: {} as never }); - expect(mocks.enqueueDelivery).toHaveBeenCalledTimes(1); - expect(mocks.deliverOutboundPayloads).toHaveBeenCalledTimes(45); + expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledTimes(1); + expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce(); + expectMockCallFields(mocks.deliverOutboundPayloads, { + skipQueue: true, + deliveryQueueId: "restart-sentinel-notice:agent:main:main:123", + }); expect(mocks.ackDelivery).not.toHaveBeenCalled(); - expect(mocks.failDelivery).toHaveBeenCalledWith("queue-1", "transport still not ready"); + expect(mocks.failDelivery).toHaveBeenCalledWith( + "restart-sentinel-notice:agent:main:main:123", + "platform outcome unknown", + ); + expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce(); + const drain = expectRecordFields(mockCallArg(mocks.drainPendingDeliveries), { + drainKey: "restart-recovery:restart-sentinel-notice:agent:main:main:123", + deliver: expect.any(Function), + }); + const selectEntry = drain.selectEntry as (entry: { id: string }) => { + match: boolean; + bypassBackoff?: boolean; + }; + expect(selectEntry({ id: "restart-sentinel-notice:agent:main:main:123" })).toEqual({ + match: true, + bypassBackoff: true, + }); + expect(selectEntry({ id: "other" })).toEqual({ match: false, bypassBackoff: true }); + expect(mocks.enqueueSystemEvent).toHaveBeenCalledTimes(1); + expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1); + expect(mocks.logWarn).toHaveBeenCalledWith( + "restart summary: outbound delivery failed; queued for recovery: Error: platform outcome unknown", + { + channel: "whatsapp", + to: "+15550002", + sessionKey: "agent:main:main", + }, + ); }); - it("still dispatches continuation after restart notice retries are exhausted", async () => { - vi.useFakeTimers(); - mocks.deliverOutboundPayloads.mockRejectedValue(new Error("transport still not ready")); + it("starts a terminal drain when the persisted retry budget is exhausted", async () => { + mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport unavailable")); + mocks.loadPendingDelivery.mockResolvedValue({ + id: "restart-sentinel-notice:agent:main:main:123", + retryCount: 45, + attemptCount: 45, + lastError: "transport unavailable", + } as never); + mocks.drainPendingDeliveries.mockImplementationOnce(async () => { + mocks.loadPendingDelivery.mockResolvedValue(null); + }); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce(); + }); + + it("preserves a notice whose persisted retry budget is not exhausted", async () => { + mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport unavailable")); + mocks.loadPendingDelivery.mockResolvedValue({ + id: "restart-sentinel-notice:agent:main:main:123", + retryCount: 7, + lastError: "database busy", + } as never); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.drainPendingDeliveries).toHaveBeenCalledTimes(46); + expect(mocks.failPendingDelivery).not.toHaveBeenCalled(); + expect(mocks.logWarn).toHaveBeenCalledWith( + "restart summary: restart notice remains queued after bounded recovery", + { + queueId: "restart-sentinel-notice:agent:main:main:123", + sessionKey: "agent:main:main", + retryCount: 7, + attemptCount: null, + maxAttempts: 45, + }, + ); + }); + + it("continues exact-id recovery after another owner releases the notice", async () => { + mocks.withActiveDeliveryClaim.mockResolvedValueOnce({ + status: "claimed-by-other-owner", + } as never); + mocks.loadPendingDelivery + .mockResolvedValueOnce({ + id: "restart-sentinel-notice:agent:main:main:123", + retryCount: 0, + } as never) + .mockResolvedValue(null); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled(); + expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce(); + expect(mocks.logInfo).toHaveBeenCalledWith( + "restart summary: durable restart notice claimed by recovery", + { sessionKey: "agent:main:main" }, + ); + }); + + it("schedules safe recovery when the delivered notice cannot be acknowledged", async () => { + mocks.ackDelivery.mockRejectedValueOnce(new Error("ack unavailable")); + mocks.loadPendingDelivery + .mockResolvedValueOnce({ + id: "restart-sentinel-notice:agent:main:main:123", + retryCount: 1, + recoveryState: "unknown_after_send", + } as never) + .mockResolvedValue(null); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.failDeliveryAfterPlatformSend).toHaveBeenCalledWith( + "restart-sentinel-notice:agent:main:main:123", + "ack unavailable", + ); + expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce(); + expect(mocks.logWarn).toHaveBeenCalledWith( + "restart summary: outbound delivery ack failed; queued for recovery: ack unavailable", + { + channel: "whatsapp", + to: "+15550002", + sessionKey: "agent:main:main", + }, + ); + }); + + it("keeps one queued restart notice when outbound delivery fails", async () => { + mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport still not ready")); + + await scheduleRestartSentinelWake({ deps: {} as never }); + + expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledTimes(1); + expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce(); + expect(mocks.ackDelivery).not.toHaveBeenCalled(); + expect(mocks.failDelivery).toHaveBeenCalledWith( + "restart-sentinel-notice:agent:main:main:123", + "transport still not ready", + ); + }); + + it("still dispatches continuation after a restart notice is queued for recovery", async () => { + mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport still not ready")); mocks.readRestartSentinel.mockResolvedValue({ + version: 1, + revision: 123, payload: { sessionKey: "agent:main:main", deliveryContext: { @@ -627,13 +889,12 @@ describe("scheduleRestartSentinelWake", () => { }, } as unknown as Awaited>); - const wakePromise = scheduleRestartSentinelWake({ deps: {} as never }); - await Promise.resolve(); - await Promise.resolve(); - await vi.advanceTimersByTimeAsync(44_000); - await wakePromise; + await scheduleRestartSentinelWake({ deps: {} as never }); - expect(mocks.failDelivery).toHaveBeenCalledWith("queue-1", "transport still not ready"); + expect(mocks.failDelivery).toHaveBeenCalledWith( + "restart-sentinel-notice:agent:main:main:123", + "transport still not ready", + ); expect(mocks.recordInboundSessionAndDispatchReply).toHaveBeenCalledTimes(1); expectContinuationDispatchFields({ routeSessionKey: "agent:main:main" }, { Body: "continue" }); }); @@ -693,7 +954,7 @@ describe("scheduleRestartSentinelWake", () => { await scheduleRestartSentinelWake({ deps: {} as never }); - expectMockCallFields(mocks.enqueueDelivery, { + expectMockCallFields(mocks.enqueueDeliveryOnce, { payloads: [{ text: "restart message" }], threadId: "thread-42", }); @@ -2114,6 +2375,8 @@ describe("scheduleRestartSentinelWake", () => { it("dispatches agentTurn continuation for a completed run entry", async () => { mocks.readRestartSentinel.mockResolvedValue({ + version: 1, + revision: 123, payload: { sessionKey: "agent:main:main", deliveryContext: { @@ -2153,6 +2416,7 @@ describe("scheduleRestartSentinelWake", () => { message: "continue after restart", messageId: "restart-sentinel:agent:main:main:agentTurn:123", expectedSessionId: "agent:main:main", + completionRetention: "permanent", route: { channel: "whatsapp", to: "+15550002", @@ -2649,6 +2913,8 @@ describe("scheduleRestartSentinelWake", () => { const busyReply = "⚠️ Previous run is still shutting down. Please try again in a moment."; let attempt = 0; mocks.readRestartSentinel.mockResolvedValue({ + version: 1, + revision: 123, payload: { sessionKey: "agent:main:main", deliveryContext: { @@ -2766,7 +3032,7 @@ describe("scheduleRestartSentinelWake", () => { await scheduleRestartSentinelWake({ deps: {} as never }); - expect(mocks.clearRestartSentinel).not.toHaveBeenCalled(); + expect(mocks.clearRestartSentinelIfRevision).not.toHaveBeenCalled(); expect(mocks.drainPendingSessionDeliveries).not.toHaveBeenCalled(); expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", { source: "restart-sentinel", @@ -2825,12 +3091,13 @@ describe("scheduleRestartSentinelWake", () => { }; mocks.readRestartSentinel.mockResolvedValue({ version: 1, + revision: 123, payload, }); await scheduleRestartSentinelWake({ deps: {} as never }); - expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce(); + expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledOnce(); expect(getLatestUpdateRestartSentinel()).toEqual(payload); }); @@ -2847,6 +3114,7 @@ describe("scheduleRestartSentinelWake", () => { }; mocks.readRestartSentinel.mockResolvedValue({ version: 1, + revision: 123, payload, }); @@ -2856,7 +3124,7 @@ describe("scheduleRestartSentinelWake", () => { expect(getLatestUpdateRestartSentinel()).toEqual(payload); }); - it("does not wake the main session when the sentinel has no sessionKey", async () => { + it("durably wakes the main session when the sentinel has no sessionKey", async () => { mocks.readRestartSentinel.mockResolvedValue({ payload: { message: "restart message", @@ -2868,7 +3136,12 @@ describe("scheduleRestartSentinelWake", () => { expect(mocks.enqueueSystemEvent).toHaveBeenCalledWith("restart message", { sessionKey: "agent:main:main", }); - expect(mocks.requestHeartbeat).not.toHaveBeenCalled(); + expect(mocks.requestHeartbeat).toHaveBeenCalledWith({ + source: "restart-sentinel", + intent: "immediate", + reason: "wake", + sessionKey: "agent:main:main", + }); expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled(); }); @@ -2910,6 +3183,15 @@ describe("scheduleRestartSentinelWake", () => { threadId: undefined, }); mocks.deliveryContextFromSession.mockReturnValue(undefined); + mocks.loadSessionEntry.mockReturnValue({ + cfg: {}, + entry: { sessionId: "agent:main:matrix:channel:!lowercased:example.org", updatedAt: 0 }, + store: {}, + storePath: "/tmp/sessions.json", + canonicalKey: "agent:main:matrix:channel:!lowercased:example.org", + storeKeys: ["agent:main:matrix:channel:!lowercased:example.org"], + legacyKey: undefined, + }); await scheduleRestartSentinelWake({ deps: {} as never }); @@ -2918,7 +3200,7 @@ describe("scheduleRestartSentinelWake", () => { sessionKey: "agent:main:matrix:channel:!lowercased:example.org", }); expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled(); - expect(mocks.enqueueDelivery).not.toHaveBeenCalled(); + expect(mocks.enqueueDeliveryOnce).not.toHaveBeenCalled(); expect(mocks.resolveOutboundTarget).not.toHaveBeenCalled(); }); diff --git a/src/gateway/server-restart-sentinel.ts b/src/gateway/server-restart-sentinel.ts index 6c3e424bd63f..ac4b5f668c94 100644 --- a/src/gateway/server-restart-sentinel.ts +++ b/src/gateway/server-restart-sentinel.ts @@ -5,7 +5,6 @@ import { REPLY_RUN_STILL_SHUTTING_DOWN_TEXT } from "../auto-reply/reply/get-repl import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js"; import { dispatchReplyWithBufferedBlockDispatcher } from "../auto-reply/reply/provider-dispatcher.js"; import type { ChatType } from "../channels/chat-type.js"; -import { sendDurableMessageBatch } from "../channels/message/runtime.js"; import { getChannelPlugin, normalizeChannelId } from "../channels/plugins/index.js"; import { recordInboundSession } from "../channels/session.js"; import { dispatchAssembledChannelTurn } from "../channels/turn/kernel.js"; @@ -14,11 +13,9 @@ import { resolveMainSessionKeyFromConfig } from "../config/sessions.js"; import { parseSessionThreadInfo } from "../config/sessions/thread-info.js"; import { formatErrorMessage, toErrorObject } from "../infra/errors.js"; import { requestHeartbeat } from "../infra/heartbeat-wake.js"; -import { ackDelivery, enqueueDelivery, failDelivery } from "../infra/outbound/delivery-queue.js"; -import { buildOutboundSessionContext } from "../infra/outbound/session-context.js"; import { resolveOutboundTarget } from "../infra/outbound/targets.js"; import { - clearRestartSentinel, + clearRestartSentinelIfRevision, finalizeUpdateRestartSentinelRunningVersion, formatRestartSentinelMessage, readRestartSentinel, @@ -53,12 +50,14 @@ import { } from "../utils/delivery-context.shared.js"; import { INTERNAL_MESSAGE_CHANNEL } from "../utils/message-channel.js"; import { deliverQueuedGeneratedMediaAgentTurn } from "./server-restart-sentinel-agent-delivery.js"; +import { + deliverRestartSentinelNotice, + enqueueRestartSentinelNotice, +} from "./server-restart-sentinel-notice.js"; import { loadSessionEntry } from "./session-utils.js"; import { runStartupTasks, type StartupTask } from "./startup-tasks.js"; const log = createSubsystemLogger("gateway/restart-sentinel"); -const OUTBOUND_RETRY_DELAY_MS = 1_000; -const OUTBOUND_MAX_ATTEMPTS = 45; const RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS = process.env.VITEST ? 1 : 6_000; const RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS = 20; const CONTROL_PLANE_UPDATE_PENDING_RETRY_DELAY_MS = process.env.VITEST ? 1 : 2_000; @@ -106,89 +105,19 @@ function enqueueRestartSentinelWake( requestHeartbeat({ source: "restart-sentinel", intent: "immediate", reason: "wake", sessionKey }); } -async function waitForOutboundRetry(delayMs: number) { +async function waitForRetry(delayMs: number) { await new Promise((resolve) => { const timer = setTimeout(resolve, delayMs); timer.unref?.(); }); } -async function deliverRestartSentinelNotice(params: { - deps: CliDeps; - cfg: ReturnType["cfg"]; - sessionKey: string; - summary: string; - message: string; - channel: string; - to: string; - accountId?: string; - replyToId?: string; - threadId?: string; - session: ReturnType; -}) { - const payloads = [{ text: params.message }]; - const queueId = await enqueueDelivery({ - channel: params.channel, - to: params.to, - accountId: params.accountId, - replyToId: params.replyToId, - threadId: params.threadId, - payloads, - bestEffort: false, - }).catch(() => null); - for (let attempt = 1; attempt <= OUTBOUND_MAX_ATTEMPTS; attempt += 1) { - try { - const send = await sendDurableMessageBatch({ - cfg: params.cfg, - channel: params.channel, - to: params.to, - accountId: params.accountId, - replyToId: params.replyToId, - threadId: params.threadId, - payloads, - session: params.session, - deps: params.deps, - bestEffort: false, - skipQueue: true, - }); - if (send.status === "failed" || send.status === "partial_failed") { - throw send.error; - } - const results = send.status === "sent" ? send.results : []; - if (results.length > 0) { - if (queueId) { - await ackDelivery(queueId).catch(() => {}); - } - return; - } - throw new Error("outbound delivery returned no results"); - } catch (err) { - const retrying = attempt < OUTBOUND_MAX_ATTEMPTS; - const suffix = retrying ? `; retrying in ${OUTBOUND_RETRY_DELAY_MS}ms` : ""; - log.warn(`${params.summary}: outbound delivery failed${suffix}: ${String(err)}`, { - channel: params.channel, - to: params.to, - sessionKey: params.sessionKey, - attempt, - maxAttempts: OUTBOUND_MAX_ATTEMPTS, - }); - if (!retrying) { - if (queueId) { - await failDelivery(queueId, formatErrorMessage(err)).catch(() => undefined); - } - return; - } - await waitForOutboundRetry(OUTBOUND_RETRY_DELAY_MS); - } - } -} - function buildRestartContinuationMessageId(params: { sessionKey: string; kind: RestartSentinelContinuation["kind"]; - ts: number; + revision: number; }) { - return `restart-sentinel:${params.sessionKey}:${params.kind}:${params.ts}`; + return `restart-sentinel:${params.sessionKey}:${params.kind}:${params.revision}`; } function resolveRestartContinuationRoute(params: { @@ -405,19 +334,22 @@ function buildQueuedRestartContinuation(params: { continuation: RestartSentinelContinuation; route?: SessionDeliveryRoute; expectedSessionId?: string | undefined; - ts: number; + revision: number; deliveryContext?: { channel?: string; to?: string; accountId?: string; threadId?: string | number; }; + idempotencyKey?: string; }): QueuedSessionDeliveryPayload { - const idempotencyKey = buildRestartContinuationMessageId({ - sessionKey: params.sessionKey, - kind: params.continuation.kind, - ts: params.ts, - }); + const idempotencyKey = + params.idempotencyKey ?? + buildRestartContinuationMessageId({ + sessionKey: params.sessionKey, + kind: params.continuation.kind, + revision: params.revision, + }); if (params.continuation.kind === "systemEvent") { return { kind: "systemEvent", @@ -426,6 +358,7 @@ function buildQueuedRestartContinuation(params: { ...(params.deliveryContext ? { deliveryContext: params.deliveryContext } : {}), idempotencyKey, maxRetries: RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS, + completionRetention: "permanent", }; } return { @@ -435,6 +368,7 @@ function buildQueuedRestartContinuation(params: { messageId: idempotencyKey, ...(params.expectedSessionId ? { expectedSessionId: params.expectedSessionId } : {}), maxRetries: RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS, + completionRetention: "permanent", ...(params.route ? { route: params.route } : {}), ...(params.deliveryContext ? { deliveryContext: params.deliveryContext } : {}), idempotencyKey, @@ -474,7 +408,7 @@ async function drainRestartContinuationQueue(params: { params.log.info( `restart continuation: entry ${params.entryId} still waiting for the previous run to clear; retrying in ${RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS}ms`, ); - await waitForOutboundRetry(RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS); + await waitForRetry(RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS); } } @@ -505,6 +439,7 @@ async function loadRestartSentinelStartupTask(params: { return null; } const payload = sentinel.payload; + const sentinelRevision = sentinel.revision; if (payload.kind === "update") { recordLatestUpdateRestartSentinel(payload); } @@ -543,14 +478,25 @@ async function loadRestartSentinelStartupTask(params: { if (!sessionKey) { const mainSessionKey = resolveMainSessionKeyFromConfig(); - enqueueSystemEvent(message, { sessionKey: mainSessionKey }); + const wakeQueueId = await enqueueSessionDelivery( + buildQueuedRestartContinuation({ + sessionKey: mainSessionKey, + continuation: { kind: "systemEvent", text: message }, + revision: sentinelRevision, + idempotencyKey: `restart-sentinel-wake:${mainSessionKey}:${sentinelRevision}`, + }), + ); if (payload.continuation) { log.warn(`${summary}: continuation skipped: restart sentinel sessionKey unavailable`, { sessionKey: mainSessionKey, continuationKind: payload.continuation.kind, }); } - await clearRestartSentinel(); + const consumed = await clearRestartSentinelIfRevision(sentinelRevision); + if (!consumed) { + log.info(`${summary}: newer restart sentinel preserved while draining durable wake`); + } + await drainRestartContinuationQueue({ deps: params.deps, entryId: wakeQueueId, log }); return { status: "ran" as const }; } @@ -587,6 +533,9 @@ async function loadRestartSentinelStartupTask(params: { let replyToId: string | undefined; let resolvedThreadId = threadId; let continuationQueueId: string | undefined; + let wakeQueueId: string | undefined; + let noticeQueueId: string | undefined; + let noticeQueueCreated = false; let continuationRoute: SessionDeliveryRoute | undefined; if (channel && to) { @@ -624,11 +573,28 @@ async function loadRestartSentinelStartupTask(params: { threadId: resolvedThreadId, chatType, }); + } + + const routedAgentTurnContinuation = + payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined; + if (!routedAgentTurnContinuation) { + wakeQueueId = await enqueueSessionDelivery( + buildQueuedRestartContinuation({ + sessionKey: canonicalKey, + continuation: { kind: "systemEvent", text: message }, + revision: sentinelRevision, + deliveryContext: wakeDeliveryContext, + idempotencyKey: `restart-sentinel-wake:${canonicalKey}:${sentinelRevision}`, + }), + ); + } + + if (payload.continuation) { continuationQueueId = await enqueueSessionDelivery( buildQueuedRestartContinuation({ sessionKey: canonicalKey, continuation: payload.continuation, - ts: payload.ts, + revision: sentinelRevision, route: continuationRoute, expectedSessionId: entry?.sessionId, deliveryContext: @@ -644,19 +610,35 @@ async function loadRestartSentinelStartupTask(params: { ); } - await clearRestartSentinel(); - const routedAgentTurnContinuation = - payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined; - if (!routedAgentTurnContinuation) { - enqueueRestartSentinelWake(message, sessionKey, wakeDeliveryContext); + if (resolvedTo && channel) { + const queuedNotice = await enqueueRestartSentinelNotice({ + channel, + to: resolvedTo, + accountId: origin?.accountId, + replyToId, + threadId: resolvedThreadId, + message, + sessionKey: canonicalKey, + revision: sentinelRevision, + }); + noticeQueueId = queuedNotice.id; + noticeQueueCreated = queuedNotice.created; } - if (resolvedTo && channel) { - const outboundSession = buildOutboundSessionContext({ - cfg, + // Every downstream intent is durable before consuming the singleton. A + // failed or stale compare-delete cannot lose work or remove a newer row. + const consumed = await clearRestartSentinelIfRevision(sentinelRevision); + if (!consumed) { + log.info(`${summary}: newer restart sentinel preserved while draining durable work`, { sessionKey: canonicalKey, }); + } + if (wakeQueueId) { + await drainRestartContinuationQueue({ deps: params.deps, entryId: wakeQueueId, log }); + } + + if (resolvedTo && channel && noticeQueueId && noticeQueueCreated) { await deliverRestartSentinelNotice({ deps: params.deps, cfg, @@ -668,7 +650,11 @@ async function loadRestartSentinelStartupTask(params: { accountId: origin?.accountId, replyToId, threadId: resolvedThreadId, - session: outboundSession, + queueId: noticeQueueId, + }); + } else if (noticeQueueId && !noticeQueueCreated) { + log.info(`${summary}: durable restart notice already owned`, { + sessionKey: canonicalKey, }); } diff --git a/src/infra/delivery-queue-sqlite.test.ts b/src/infra/delivery-queue-sqlite.test.ts index b506549b5e10..10e6ee9ea73c 100644 --- a/src/infra/delivery-queue-sqlite.test.ts +++ b/src/infra/delivery-queue-sqlite.test.ts @@ -220,6 +220,45 @@ describe("delivery-queue-sqlite corrupt JSON resilience", () => { last_error: null, }); }); + + it("never prunes a permanent producer receipt", () => { + upsertDeliveryQueueEntry({ + queueName: QUEUE, + entry: { + id: "rt-permanent", + enqueuedAt: 1, + retryCount: 0, + completionRetention: "permanent", + }, + stateDir, + }); + completeDeliveryQueueEntry(QUEUE, "rt-permanent", stateDir); + const { db } = openOpenClawStateDatabase({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + db.prepare( + `UPDATE delivery_queue_entries + SET enqueued_at = ? + WHERE queue_name = ? AND id = ?`, + ).run(Date.now() - 31 * 24 * 60 * 60_000, QUEUE, "rt-permanent"); + + enqueueValid("rt-prune-trigger"); + completeDeliveryQueueEntry(QUEUE, "rt-prune-trigger", stateDir); + + expect(getDeliveryQueueEntryStatus(QUEUE, "rt-permanent", stateDir)).toBe("completed"); + const row = db + .prepare( + `SELECT recovery_state, entry_json + FROM delivery_queue_entries + WHERE queue_name = ? AND id = ?`, + ) + .get(QUEUE, "rt-permanent") as Record; + expect(row.recovery_state).toBe("completed_permanent"); + expect(JSON.parse(String(row.entry_json))).toMatchObject({ + completionRetention: "permanent", + recoveryState: "completed_permanent", + }); + }); }); }); diff --git a/src/infra/delivery-queue-sqlite.ts b/src/infra/delivery-queue-sqlite.ts index ee5438ab8be9..964c74bff83d 100644 --- a/src/infra/delivery-queue-sqlite.ts +++ b/src/infra/delivery-queue-sqlite.ts @@ -13,6 +13,9 @@ import { runSqliteImmediateTransactionSync } from "./sqlite-transaction.js"; type QueueStatus = "pending" | "failed" | "completed"; type DeliveryQueueDatabase = Pick; const COMPLETED_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000; +const PERMANENT_COMPLETION_RECOVERY_STATE = "completed_permanent"; + +export type DeliveryQueueCompletionRetention = "permanent"; /** Indexed metadata extracted from queue payloads for diagnostics and recovery. */ export type DeliveryQueueRowMetadata = { @@ -28,6 +31,9 @@ export type DeliveryQueueEntryState = { id: string; enqueuedAt: number; retryCount: number; + /** Durable delivery-call count reserved before invoking the provider path. */ + attemptCount?: number; + completionRetention?: DeliveryQueueCompletionRetention; acknowledgedAt?: number; lastAttemptAt?: number; lastError?: string; @@ -433,11 +439,19 @@ export function deleteDeliveryQueueEntry(queueName: string, id: string, stateDir /** Retain a delivered row as a durable idempotency tombstone. */ export function completeDeliveryQueueEntry(queueName: string, id: string, stateDir?: string): void { const now = Date.now(); + const current = loadDeliveryQueueEntry(queueName, id, stateDir); + const retainPermanently = current?.completionRetention === "permanent"; const tombstone = { id, enqueuedAt: now, retryCount: 0, acknowledgedAt: now, + ...(retainPermanently + ? { + completionRetention: "permanent" as const, + recoveryState: PERMANENT_COMPLETION_RECOVERY_STATE, + } + : {}), }; const completed = upsertDeliveryQueueEntry({ queueName, @@ -453,7 +467,8 @@ export function completeDeliveryQueueEntry(queueName: string, id: string, stateD } throw enoent(queueName, id); } - // Thirty days covers delayed producer replays while bounding successful-row growth. + // Ordinary receipts expire after thirty days. Permanent producer receipts + // survive because their source intent can outlive any bounded retry window. const database = openStateDatabase(stateDir); const queueDb = getNodeSqliteKysely(database.db); executeSqliteQuerySync( @@ -462,7 +477,13 @@ export function completeDeliveryQueueEntry(queueName: string, id: string, stateD .deleteFrom("delivery_queue_entries") .where("queue_name", "=", queueName) .where("status", "=", "completed") - .where("enqueued_at", "<", now - COMPLETED_TOMBSTONE_RETENTION_MS), + .where("enqueued_at", "<", now - COMPLETED_TOMBSTONE_RETENTION_MS) + .where((eb) => + eb.or([ + eb("recovery_state", "is", null), + eb("recovery_state", "!=", PERMANENT_COMPLETION_RECOVERY_STATE), + ]), + ), ); } @@ -480,6 +501,59 @@ export function updateDeliveryQueueEntry( upsertDeliveryQueueEntry({ queueName, entry: update(current), stateDir }); } +type ReserveDeliveryQueueAttemptResult = + | { status: "reserved"; attemptCount: number } + | { status: "exhausted"; attemptCount: number }; + +/** Atomically reserve one provider-delivery call before executing it. */ +export function reserveDeliveryQueueEntryAttempt(params: { + queueName: string; + id: string; + maxAttempts: number; + stateDir?: string; +}): ReserveDeliveryQueueAttemptResult { + if (!Number.isInteger(params.maxAttempts) || params.maxAttempts <= 0) { + throw new Error(`Invalid delivery attempt budget: ${params.maxAttempts}`); + } + const database = openStateDatabase(params.stateDir); + return runSqliteImmediateTransactionSync( + database.db, + () => { + const current = loadDeliveryQueueEntry(params.queueName, params.id, params.stateDir); + if (!current) { + throw enoent(params.queueName, params.id); + } + const persistedAttemptCount = + typeof current.attemptCount === "number" && + Number.isInteger(current.attemptCount) && + current.attemptCount >= 0 + ? current.attemptCount + : 0; + const attemptCount = Math.max(persistedAttemptCount, current.retryCount); + if (attemptCount >= params.maxAttempts) { + return { status: "exhausted", attemptCount }; + } + const reservedAttemptCount = attemptCount + 1; + const updated = upsertDeliveryQueueEntryInDatabase( + { + queueName: params.queueName, + entry: { ...current, attemptCount: reservedAttemptCount }, + updatePendingOnly: true, + }, + database, + ); + if (!updated) { + throw enoent(params.queueName, params.id); + } + return { status: "reserved", attemptCount: reservedAttemptCount }; + }, + { + databaseLabel: "openclaw-state", + operationLabel: `reserve ${params.queueName} delivery attempt`, + }, + ); +} + /** Dead-lettered entry counts for one queue namespace. */ type FailedDeliveryQueueCount = { queueName: string; diff --git a/src/infra/outbound/delivery-queue-recovery.ts b/src/infra/outbound/delivery-queue-recovery.ts index 5a2d2c0f84c4..937d6b6f59da 100644 --- a/src/infra/outbound/delivery-queue-recovery.ts +++ b/src/infra/outbound/delivery-queue-recovery.ts @@ -52,6 +52,7 @@ import { loadPendingDeliveries, markDeliveryPlatformOutcomeUnknown, moveToFailed, + reserveDeliveryAttempt, type QueuedDelivery, type QueuedDeliveryPayload, } from "./delivery-queue-storage.js"; @@ -98,7 +99,7 @@ type ActiveDeliveryClaimResult = | { status: "claimed"; value: T } | { status: "claimed-by-other-owner" }; -const MAX_RETRIES = 5; +const DEFAULT_MAX_RETRIES = 5; const PERMANENT_ERROR_PATTERNS: readonly RegExp[] = [ /no conversation reference found/i, @@ -118,6 +119,20 @@ const drainInProgress = new Map(); const entriesInProgress = new Set(); const recoveryReplayPacer = createRecoveryReplayPacer(); +function resolveMaxRetries(entry: QueuedDelivery): number { + const configured = entry.maxRetries; + return typeof configured === "number" && Number.isInteger(configured) && configured > 0 + ? configured + : DEFAULT_MAX_RETRIES; +} + +function resolveAttemptCount(entry: QueuedDelivery): number { + const persisted = entry.attemptCount; + const attemptCount = + typeof persisted === "number" && Number.isInteger(persisted) && persisted >= 0 ? persisted : 0; + return Math.max(attemptCount, entry.retryCount); +} + function resolveRecoveryDeadlineMs(maxRecoveryMs: number | undefined): number { const durationMs = typeof maxRecoveryMs === "number" && Number.isFinite(maxRecoveryMs) @@ -150,10 +165,14 @@ function emitQueuedAuditTerminals( }); } +function needsUnknownSendReconciliation(entry: QueuedDelivery): boolean { + return ( + entry.recoveryState === "send_attempt_started" || entry.recoveryState === "unknown_after_send" + ); +} + function queuedDeadLetterAuditTerminals(entry: QueuedDelivery) { - const ambiguous = - entry.recoveryState === "send_attempt_started" || entry.recoveryState === "unknown_after_send"; - if (ambiguous) { + if (needsUnknownSendReconciliation(entry)) { return uniformOutboundAuditTerminals(entry.payloads.length, { outcome: "unknown", failureStage: "queue", @@ -597,14 +616,13 @@ async function drainQueuedEntry(opts: { onFailed?: (entry: QueuedDelivery, errMsg: string) => void; }): Promise<"recovered" | "failed" | "moved-to-failed" | "already-gone"> { const { entry } = opts; + const maxRetries = resolveMaxRetries(entry); + const attemptBudgetExhausted = resolveAttemptCount(entry) >= maxRetries; const ownerState = await resolveCompletedOwnerBeforeRecovery(opts); if (ownerState !== "continue") { return ownerState; } - if ( - entry.recoveryState === "send_attempt_started" || - entry.recoveryState === "unknown_after_send" - ) { + if (needsUnknownSendReconciliation(entry)) { // A crash after platform send start cannot be blindly replayed; adapters // must reconcile whether the platform already committed the message. const reconciliation = await reconcileUnknownQueuedDelivery({ @@ -668,7 +686,11 @@ async function drainQueuedEntry(opts: { } opts.log.warn(`Delivery entry ${entry.id} ${errMsg}`); opts.onFailed?.(entry, errMsg); - if (reconciliation?.status === "unresolved" && reconciliation.retryable === true) { + if ( + reconciliation?.status === "unresolved" && + reconciliation.retryable === true && + !attemptBudgetExhausted + ) { try { await failDelivery(entry.id, errMsg, opts.stateDir); return "failed"; @@ -715,6 +737,22 @@ async function drainQueuedEntry(opts: { commitHooksRun = true; await runOutboundDeliveryCommitHooks(deliveredResults); }; + const reservation = await reserveDeliveryAttempt(entry.id, maxRetries, opts.stateDir); + if (reservation.status === "exhausted") { + const errMsg = `delivery retry budget exhausted (${reservation.attemptCount}/${maxRetries})`; + markDurableDeliveryFailedBestEffort(entry, opts.log); + try { + await moveToFailed(entry.id, opts.stateDir); + } catch (moveErr) { + if (getErrnoCode(moveErr) === "ENOENT") { + return "already-gone"; + } + throw moveErr; + } + emitQueuedAuditTerminals(entry, () => queuedDeadLetterAuditTerminals(entry)); + opts.onFailed?.(entry, errMsg); + return "moved-to-failed"; + } const recoverySpoolPaths = collectEntrySpoolPaths(entry.payloads, opts.stateDir); let mediaRecoveryLeaseId: string | undefined; try { @@ -980,7 +1018,11 @@ export async function drainPendingDeliveries(opts: { continue; } - if (currentEntry.retryCount >= MAX_RETRIES) { + const maxRetries = resolveMaxRetries(currentEntry); + if ( + resolveAttemptCount(currentEntry) >= maxRetries && + !needsUnknownSendReconciliation(currentEntry) + ) { try { markDurableDeliveryFailedBestEffort(currentEntry, opts.log); await moveToFailed(currentEntry.id, opts.stateDir); @@ -1044,7 +1086,7 @@ export async function drainPendingDeliveries(opts: { /** * On gateway startup, scan the delivery queue and retry any pending entries. - * Uses exponential backoff and moves entries that exceed MAX_RETRIES to failed/. + * Uses exponential backoff and moves entries that exhaust their retry budget to failed/. */ export async function recoverPendingDeliveries(opts: { deliver: DeliverFn; @@ -1100,9 +1142,11 @@ export async function recoverPendingDeliveries(opts: { continue; } - if (currentEntry.retryCount >= MAX_RETRIES) { + const maxRetries = resolveMaxRetries(currentEntry); + const attemptCount = resolveAttemptCount(currentEntry); + if (attemptCount >= maxRetries && !needsUnknownSendReconciliation(currentEntry)) { opts.log.warn( - `Delivery ${currentEntry.id} exceeded max retries (${currentEntry.retryCount}/${MAX_RETRIES}) — moving to failed/`, + `Delivery ${currentEntry.id} exceeded max retries (${attemptCount}/${maxRetries}) — moving to failed/`, ); const movedToFailed = await moveEntryToFailedWithLogging( currentEntry, diff --git a/src/infra/outbound/delivery-queue-storage.ts b/src/infra/outbound/delivery-queue-storage.ts index b6c9d068610e..582e4486bffb 100644 --- a/src/infra/outbound/delivery-queue-storage.ts +++ b/src/infra/outbound/delivery-queue-storage.ts @@ -6,6 +6,7 @@ import type { RenderedMessageBatchPlanItem } from "../../channels/message/types. import type { ReplyToMode } from "../../config/types.js"; import type { PluginHookReplyPayloadSendingContext } from "../../plugins/hook-types.js"; import { + completeDeliveryQueueEntry, commitStagedDeliveryQueueEntry, commitStagedDeliveryQueueEntryOnce, deleteDeliveryQueueEntry, @@ -13,9 +14,11 @@ import { loadDeliveryQueueEntries, loadDeliveryQueueEntry, moveDeliveryQueueEntryToFailed, + reserveDeliveryQueueEntryAttempt, updateDeliveryQueueEntry, upsertDeliveryQueueEntry, type DeliveryQueueRowMetadata, + type DeliveryQueueCompletionRetention, } from "../delivery-queue-sqlite.js"; import { generateSecureUuid } from "../secure-random.js"; import type { DurableDeliveryCompletion } from "./delivery-completion.js"; @@ -85,12 +88,17 @@ export type QueuedDeliveryPayload = { preparedMessageId?: string; /** Serializable owner state finalized by both live delivery and recovery. */ deliveryCompletion?: DurableDeliveryCompletion; + /** Retain a terminal receipt when the producer may replay this stable intent indefinitely. */ + completionRetention?: DeliveryQueueCompletionRetention; + /** Producer-specific retry budget; omitted entries use the queue default. */ + maxRetries?: number; }; export interface QueuedDelivery extends QueuedDeliveryPayload { id: string; enqueuedAt: number; retryCount: number; + attemptCount: number; lastAttemptAt?: number; lastError?: string; platformSendStartedAt?: number; @@ -135,7 +143,10 @@ function createQueuedDelivery(params: QueuedDeliveryPayload, id: string): Queued gatewayClientScopes: params.gatewayClientScopes, preparedMessageId: params.preparedMessageId, deliveryCompletion: params.deliveryCompletion, + completionRetention: params.completionRetention, + maxRetries: params.maxRetries, retryCount: 0, + attemptCount: 0, }; } @@ -224,7 +235,7 @@ type AckDeliveryOptions = { retainSpoolArtifacts?: boolean; }; -/** Remove a successfully delivered entry from the queue. */ +/** Remove a successfully delivered entry, or retain its permanent producer receipt. */ export async function ackDelivery( id: string, stateDir?: string, @@ -233,8 +244,17 @@ export async function ackDelivery( // Read the media references before the row goes, then unlink only after the // delete commits. A crash in between leaves an orphan for the retention sweep; // unlinking first could strip media from a row that still has to replay. - const spoolPaths = loadEntrySpoolPaths(id, stateDir); - deleteDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir); + const entry = loadDeliveryQueueEntry( + OUTBOUND_DELIVERY_QUEUE_NAME, + id, + stateDir, + ) as QueuedDelivery | null; + const spoolPaths = entry ? collectEntrySpoolPaths(entry.payloads, stateDir) : []; + if (entry?.completionRetention === "permanent") { + completeDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir); + } else { + deleteDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir); + } if (!options?.retainSpoolArtifacts) { await releaseSpoolArtifacts(spoolPaths, stateDir); } @@ -283,6 +303,16 @@ export async function failDeliveryAfterPlatformSend( })); } +/** Reserve one durable delivery call before invoking the provider path. */ +export async function reserveDeliveryAttempt(id: string, maxAttempts: number, stateDir?: string) { + return reserveDeliveryQueueEntryAttempt({ + queueName: OUTBOUND_DELIVERY_QUEUE_NAME, + id, + maxAttempts, + stateDir, + }); +} + function updateQueuedDelivery( id: string, stateDir: string | undefined, diff --git a/src/infra/outbound/delivery-queue.reconnect-drain.test.ts b/src/infra/outbound/delivery-queue.reconnect-drain.test.ts index 9ed6e6155cb2..aca95114f32b 100644 --- a/src/infra/outbound/delivery-queue.reconnect-drain.test.ts +++ b/src/infra/outbound/delivery-queue.reconnect-drain.test.ts @@ -4,13 +4,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { controlNextRecoverySleep } from "../../../test/helpers/infra/delivery-recovery.js"; import type { OpenClawConfig } from "../../config/config.js"; import { openOpenClawStateDatabase } from "../../state/openclaw-state-db.js"; -import { loadPendingDeliveries } from "./delivery-queue-storage.js"; +import { loadPendingDeliveries, reserveDeliveryAttempt } from "./delivery-queue-storage.js"; import { type DeliverFn, drainPendingDeliveries, enqueueDelivery, failDelivery, markDeliveryPlatformOutcomeUnknown, + markDeliveryPlatformSendAttemptStarted, type RecoveryLogger, recoverPendingDeliveries, withActiveDeliveryClaim, @@ -27,8 +28,12 @@ const MAX_RETRIES = 5; const stubCfg = {} as OpenClawConfig; const NO_LISTENER_ERROR = "No active DirectChat listener"; const sleepMock = vi.hoisted(() => vi.fn<(ms: number) => Promise>()); +const resolveOutboundChannelMessageAdapterMock = vi.hoisted(() => vi.fn()); vi.mock("../../utils/sleep.js", () => ({ sleep: sleepMock })); +vi.mock("./channel-resolution.js", () => ({ + resolveOutboundChannelMessageAdapter: resolveOutboundChannelMessageAdapterMock, +})); function normalizeReconnectAccountIdForTest(accountId?: string | null): string { return (accountId ?? "").trim() || "default"; @@ -146,6 +151,7 @@ describe("drainPendingDeliveries for reconnect", () => { tmpDir = fixtures.tmpDir(); sleepMock.mockReset(); sleepMock.mockResolvedValue(undefined); + resolveOutboundChannelMessageAdapterMock.mockReset(); }); it("drains entries that failed with 'no listener' error", async () => { @@ -218,6 +224,46 @@ describe("drainPendingDeliveries for reconnect", () => { expectLogMessageWith(log.warn, "refusing blind replay without adapter reconciliation"); }); + it("reconciles an exhausted final attempt before dead-lettering", async () => { + const log = createRecoveryLog(); + const deliver = vi.fn(async () => {}); + const id = await enqueueDelivery( + { + channel: "directchat", + to: "+1555", + payloads: [{ text: "maybe sent" }], + accountId: "acct1", + maxRetries: 1, + }, + tmpDir, + ); + await reserveDeliveryAttempt(id, 1, tmpDir); + await markDeliveryPlatformSendAttemptStarted(id, tmpDir); + const reconcileUnknownSend = vi.fn().mockResolvedValue({ + status: "sent", + messageId: "platform-final", + receipt: { + primaryPlatformMessageId: "platform-final", + platformMessageIds: ["platform-final"], + parts: [{ platformMessageId: "platform-final", kind: "text", index: 0 }], + sentAt: 1, + }, + }); + resolveOutboundChannelMessageAdapterMock.mockReturnValue({ + durableFinal: { + capabilities: { reconcileUnknownSend: true }, + reconcileUnknownSend, + }, + }); + + await drainAcct1DirectChatReconnect({ deliver, log, stateDir: tmpDir }); + + expect(deliver).not.toHaveBeenCalled(); + expect(reconcileUnknownSend).toHaveBeenCalledOnce(); + expect(await loadPendingDeliveries(tmpDir)).toHaveLength(0); + expect(readOutboundQueueStatus(tmpDir, id)).toBeUndefined(); + }); + it("skips entries where retryCount >= MAX_RETRIES", async () => { const log = createRecoveryLog(); const deliver = vi.fn(async () => {}); diff --git a/src/infra/outbound/delivery-queue.recovery.test.ts b/src/infra/outbound/delivery-queue.recovery.test.ts index ee05c33d10c3..b711f3c00f5f 100644 --- a/src/infra/outbound/delivery-queue.recovery.test.ts +++ b/src/infra/outbound/delivery-queue.recovery.test.ts @@ -24,7 +24,7 @@ import { } from "./deliver-types.js"; import { attachOutboundDeliveryCommitHook } from "./delivery-commit-hooks.js"; import { pruneOrphanedDeliveryQueueMedia } from "./delivery-queue-media-spool.js"; -import { loadPendingDeliveries } from "./delivery-queue-storage.js"; +import { loadPendingDeliveries, reserveDeliveryAttempt } from "./delivery-queue-storage.js"; import { ackDelivery, enqueueDelivery, @@ -498,6 +498,67 @@ describe("delivery-queue recovery", () => { }); }); + it("honors a producer-specific retry budget", async () => { + const id = await enqueueDelivery( + { + channel: "demo-channel-a", + to: "+1", + payloads: [{ text: "a" }], + maxRetries: 45, + }, + tmpDir(), + ); + setQueuedEntryState(tmpDir(), id, { + retryCount: MAX_RETRIES, + lastAttemptAt: Date.now() - 10_000_000, + }); + const deliver = vi.fn().mockResolvedValue([]); + + const { result } = await runRecovery({ deliver }); + + expect(deliver).toHaveBeenCalledOnce(); + expect(result).toMatchObject({ recovered: 1, skippedMaxRetries: 0 }); + expect(readOutboundQueueStatus(tmpDir(), id)).toBeUndefined(); + }); + + it("dead-letters an atomically exhausted attempt budget before replay", async () => { + const id = await enqueueDelivery( + { + channel: "demo-channel-a", + to: "+1", + payloads: [{ text: "a" }], + maxRetries: 1, + }, + tmpDir(), + ); + await reserveDeliveryAttempt(id, 1, tmpDir()); + const deliver = vi.fn(); + + const { result } = await runRecovery({ deliver }); + + expect(deliver).not.toHaveBeenCalled(); + expect(result.skippedMaxRetries).toBe(1); + expect(readOutboundQueueStatus(tmpDir(), id)).toBe("failed"); + }); + + it("ignores an invalid producer retry budget", async () => { + await enqueueDelivery( + { + channel: "demo-channel-a", + to: "+1", + payloads: [{ text: "a" }], + maxRetries: 0.5, + }, + tmpDir(), + ); + const deliver = vi.fn().mockResolvedValue([]); + + const { result } = await runRecovery({ deliver }); + + expect(deliver).toHaveBeenCalledOnce(); + expect(result).toMatchObject({ recovered: 1, skippedMaxRetries: 0 }); + }); + it("dead-letters max-retry entries even when conversation owner state is missing", async () => { const storePath = path.join(tmpDir(), "missing-owner-sessions.json"); const id = await enqueueDelivery( @@ -537,6 +598,7 @@ describe("delivery-queue recovery", () => { ); setQueuedEntryState(tmpDir(), id, { retryCount: MAX_RETRIES, + lastAttemptAt: Date.now() - 10_000_000, platformSendStartedAt: Date.now(), recoveryState: "send_attempt_started", }); @@ -544,7 +606,7 @@ describe("delivery-queue recovery", () => { const { result } = await runRecovery({ deliver: vi.fn() }); unsubscribe(); - expect(result.skippedMaxRetries).toBe(1); + expect(result).toMatchObject({ failed: 1, skippedMaxRetries: 0 }); expect(auditEvents).toHaveLength(1); expect(auditEvents[0]).toMatchObject({ sourceId: `message:outbound:queue:${id}:payload:0`, @@ -571,6 +633,7 @@ describe("delivery-queue recovery", () => { const entries = await loadPendingDeliveries(tmpDir()); expect(entries).toHaveLength(1); expect(entries[0]?.retryCount).toBe(1); + expect(entries[0]?.attemptCount).toBe(1); expect(entries[0]?.lastError).toBe("network down"); expect(auditEvents).toEqual([]); }); @@ -1008,9 +1071,11 @@ describe("delivery-queue recovery", () => { replyToId: "root-message", threadId: "thread-1", silent: true, + maxRetries: 1, }, tmpDir(), ); + await reserveDeliveryAttempt(id, 1, tmpDir()); await markDeliveryPlatformSendAttemptStarted(id, tmpDir(), { replyToId: "hooked-root-message", }); @@ -1172,6 +1237,45 @@ describe("delivery-queue recovery", () => { expect(entries[0]?.lastError).toContain("provider lookup timed out"); }); + it("dead-letters an exhausted unknown send after retryable reconciliation fails", async () => { + const id = await enqueueDelivery( + { + channel: "demo-channel-a", + to: "+1", + payloads: [{ text: "unknown final attempt" }], + maxRetries: 1, + }, + tmpDir(), + ); + await reserveDeliveryAttempt(id, 1, tmpDir()); + setQueuedEntryState(tmpDir(), id, { + retryCount: 0, + lastAttemptAt: Date.now() - 10_000_000, + platformSendStartedAt: Date.now(), + recoveryState: "unknown_after_send", + }); + const reconcileUnknownSend = vi.fn().mockResolvedValue({ + status: "unresolved", + error: "provider lookup timed out", + retryable: true, + }); + resolveOutboundChannelMessageAdapterMock.mockReturnValue({ + durableFinal: { + capabilities: { reconcileUnknownSend: true }, + reconcileUnknownSend, + }, + }); + const deliver = vi.fn().mockResolvedValue([]); + + const { result } = await runRecovery({ deliver }); + + expect(reconcileUnknownSend).toHaveBeenCalledOnce(); + expect(deliver).not.toHaveBeenCalled(); + expect(result).toMatchObject({ failed: 1, skippedMaxRetries: 0 }); + expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0); + expect(readOutboundQueueStatus(tmpDir(), id)).toBe("failed"); + }); + it("does not reconcile unknown-after-send entries unless the adapter declares the capability", async () => { const id = await enqueueDelivery( { channel: "demo-channel-a", to: "+1", payloads: [{ text: "hidden method" }] }, diff --git a/src/infra/outbound/delivery-queue.storage.test.ts b/src/infra/outbound/delivery-queue.storage.test.ts index 014678aa2d48..7462a9832383 100644 --- a/src/infra/outbound/delivery-queue.storage.test.ts +++ b/src/infra/outbound/delivery-queue.storage.test.ts @@ -9,6 +9,7 @@ import { loadPendingDelivery, loadPendingDeliveries, moveToFailed, + reserveDeliveryAttempt, } from "./delivery-queue-storage.js"; import { ackDelivery, @@ -55,6 +56,40 @@ describe("delivery-queue storage", () => { } describe("enqueue + ack lifecycle", () => { + it("persists a producer-specific retry budget", async () => { + const id = await enqueueTextDelivery({ + channel: "directchat", + to: "+1555", + payloads: [{ text: "retry-budget" }], + maxRetries: 45, + }); + + expect(readQueuedEntry(tmpDir(), id).maxRetries).toBe(45); + }); + + it("atomically reserves delivery attempts up to the producer budget", async () => { + const id = await enqueueTextDelivery({ + channel: "directchat", + to: "+1555", + payloads: [{ text: "attempt-budget" }], + maxRetries: 2, + }); + + await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({ + status: "reserved", + attemptCount: 1, + }); + await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({ + status: "reserved", + attemptCount: 2, + }); + await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({ + status: "exhausted", + attemptCount: 2, + }); + expect(readQueuedEntry(tmpDir(), id).attemptCount).toBe(2); + }); + it("creates and removes a queue entry", async () => { const id = await enqueueTextDelivery( { @@ -180,6 +215,36 @@ describe("delivery-queue storage", () => { }); }); + it("keeps permanent completion ownership after ack", async () => { + const id = "restart-sentinel-notice:agent:main:main:123"; + await enqueueDeliveryOnce( + { + channel: "directchat", + to: "+1555", + payloads: [{ text: "restart complete" }], + completionRetention: "permanent", + }, + id, + tmpDir(), + ); + + await ackDelivery(id, tmpDir()); + const repeated = await enqueueDeliveryOnce( + { + channel: "directchat", + to: "+1555", + payloads: [{ text: "must not replay" }], + completionRetention: "permanent", + }, + id, + tmpDir(), + ); + + expect(repeated).toEqual({ id, created: false }); + expect(await loadPendingDeliveries(tmpDir())).toEqual([]); + expect(readStatus(id)).toBe("completed"); + }); + it("ack is idempotent (no error on missing file)", async () => { await expect(ackDelivery("nonexistent-id", tmpDir())).resolves.toBeUndefined(); }); diff --git a/src/infra/restart-sentinel-store.ts b/src/infra/restart-sentinel-store.ts new file mode 100644 index 000000000000..00e0f9020a08 --- /dev/null +++ b/src/infra/restart-sentinel-store.ts @@ -0,0 +1,660 @@ +import type { DatabaseSync } from "node:sqlite"; +import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; + +type RestartSentinelLog = { + stdoutTail?: string | null; + stderrTail?: string | null; + exitCode?: number | null; +}; + +type RestartSentinelStep = { + name: string; + command: string; + cwd?: string | null; + durationMs?: number | null; + log?: RestartSentinelLog | null; +}; + +type RestartSentinelStats = { + mode?: string; + root?: string; + requiresRestart?: boolean; + handoffId?: string; + before?: Record | null; + after?: Record | null; + steps?: RestartSentinelStep[]; + reason?: string | null; + durationMs?: number | null; +}; + +export type RestartSentinelContinuation = + | { + kind: "systemEvent"; + text: string; + } + | { + kind: "agentTurn"; + message: string; + }; + +export type RestartSentinelPayload = { + kind: "config-apply" | "config-auto-recovery" | "config-patch" | "update" | "restart"; + status: "ok" | "error" | "skipped"; + ts: number; + sessionKey?: string; + deliveryContext?: { + channel?: string; + to?: string; + accountId?: string; + }; + threadId?: string; + message?: string | null; + continuation?: RestartSentinelContinuation | null; + doctorHint?: string | null; + stats?: RestartSentinelStats | null; +}; + +export type RestartSentinelEnvelope = { + version: 1; + payload: RestartSentinelPayload; +}; + +export type RestartSentinel = RestartSentinelEnvelope & { + /** Optimistic-concurrency revision backed by gateway_restart_sentinel.updated_at_ms. */ + revision: number; +}; + +type RestartSentinelRowState = + | { kind: "missing" } + | { kind: "invalid"; revision: number } + | { kind: "valid"; sentinel: RestartSentinel }; + +const RESTART_SENTINEL_KEY = "current"; +const RESTART_SENTINEL_REVISION_FLOOR_KEY = "revision-floor"; +const RESTART_SENTINEL_KINDS = new Set([ + "config-apply", + "config-auto-recovery", + "config-patch", + "update", + "restart", +]); +const RESTART_SENTINEL_STATUSES = new Set([ + "ok", + "error", + "skipped", +]); + +type GatewayRestartSentinelDatabase = Pick; + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function isSafeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value); +} + +function parseOptionalNullableString( + record: Record, + key: string, +): string | null | undefined | false { + const value = record[key]; + if (value === undefined || value === null || typeof value === "string") { + return value; + } + return false; +} + +function parseRestartSentinelLog(value: unknown): RestartSentinelLog | null { + if (!isPlainRecord(value)) { + return null; + } + const stdoutTail = parseOptionalNullableString(value, "stdoutTail"); + const stderrTail = parseOptionalNullableString(value, "stderrTail"); + const exitCode = value.exitCode; + if ( + stdoutTail === false || + stderrTail === false || + (exitCode !== undefined && exitCode !== null && !isSafeInteger(exitCode)) + ) { + return null; + } + const result: RestartSentinelLog = {}; + if (stdoutTail !== undefined) { + result.stdoutTail = stdoutTail; + } + if (stderrTail !== undefined) { + result.stderrTail = stderrTail; + } + if (exitCode !== undefined) { + result.exitCode = exitCode as number | null; + } + return result; +} + +function parseRestartSentinelStep(value: unknown): RestartSentinelStep | null { + if ( + !isPlainRecord(value) || + typeof value.name !== "string" || + typeof value.command !== "string" + ) { + return null; + } + const cwd = parseOptionalNullableString(value, "cwd"); + const durationMs = value.durationMs; + const log = value.log; + if ( + cwd === false || + (durationMs !== undefined && durationMs !== null && !isFiniteNumber(durationMs)) || + (log !== undefined && log !== null && !parseRestartSentinelLog(log)) + ) { + return null; + } + const result: RestartSentinelStep = { name: value.name, command: value.command }; + if (cwd !== undefined) { + result.cwd = cwd; + } + if (durationMs !== undefined) { + result.durationMs = durationMs as number | null; + } + if (log !== undefined) { + result.log = log === null ? null : parseRestartSentinelLog(log); + } + return result; +} + +function parseRestartSentinelStats(value: unknown): RestartSentinelStats | null { + if (!isPlainRecord(value)) { + return null; + } + const mode = parseOptionalNullableString(value, "mode"); + const root = parseOptionalNullableString(value, "root"); + const handoffId = parseOptionalNullableString(value, "handoffId"); + const reason = parseOptionalNullableString(value, "reason"); + const before = value.before; + const after = value.after; + const steps = value.steps; + const durationMs = value.durationMs; + if ( + mode === false || + mode === null || + root === false || + root === null || + handoffId === false || + handoffId === null || + reason === false || + (value.requiresRestart !== undefined && typeof value.requiresRestart !== "boolean") || + (before !== undefined && before !== null && !isPlainRecord(before)) || + (after !== undefined && after !== null && !isPlainRecord(after)) || + (steps !== undefined && + (!Array.isArray(steps) || steps.some((step) => !parseRestartSentinelStep(step)))) || + (durationMs !== undefined && durationMs !== null && !isFiniteNumber(durationMs)) + ) { + return null; + } + const result: RestartSentinelStats = {}; + if (mode !== undefined) { + result.mode = mode; + } + if (root !== undefined) { + result.root = root; + } + if (value.requiresRestart !== undefined) { + result.requiresRestart = value.requiresRestart as boolean; + } + if (handoffId !== undefined) { + result.handoffId = handoffId; + } + if (before !== undefined) { + result.before = before as Record | null; + } + if (after !== undefined) { + result.after = after as Record | null; + } + if (steps !== undefined) { + result.steps = steps.map((step) => parseRestartSentinelStep(step)!); + } + if (reason !== undefined) { + result.reason = reason; + } + if (durationMs !== undefined) { + result.durationMs = durationMs as number | null; + } + return result; +} + +function parseRestartSentinelContinuation(value: unknown): RestartSentinelContinuation | null { + if (!isPlainRecord(value)) { + return null; + } + if (value.kind === "systemEvent" && typeof value.text === "string") { + return { kind: "systemEvent", text: value.text }; + } + if (value.kind === "agentTurn" && typeof value.message === "string") { + return { kind: "agentTurn", message: value.message }; + } + return null; +} + +function parseRestartSentinelPayload(value: unknown): RestartSentinelPayload | null { + if ( + !isPlainRecord(value) || + !RESTART_SENTINEL_KINDS.has(value.kind as RestartSentinelPayload["kind"]) || + !RESTART_SENTINEL_STATUSES.has(value.status as RestartSentinelPayload["status"]) || + !isSafeInteger(value.ts) + ) { + return null; + } + const sessionKey = parseOptionalNullableString(value, "sessionKey"); + const threadId = parseOptionalNullableString(value, "threadId"); + const message = parseOptionalNullableString(value, "message"); + const doctorHint = parseOptionalNullableString(value, "doctorHint"); + if ( + sessionKey === false || + sessionKey === null || + threadId === false || + threadId === null || + message === false || + doctorHint === false + ) { + return null; + } + + let deliveryContext: RestartSentinelPayload["deliveryContext"]; + if (value.deliveryContext !== undefined) { + if (!isPlainRecord(value.deliveryContext)) { + return null; + } + const channel = parseOptionalNullableString(value.deliveryContext, "channel"); + const to = parseOptionalNullableString(value.deliveryContext, "to"); + const accountId = parseOptionalNullableString(value.deliveryContext, "accountId"); + if ( + channel === false || + channel === null || + to === false || + to === null || + accountId === false || + accountId === null + ) { + return null; + } + deliveryContext = {}; + if (channel !== undefined) { + deliveryContext.channel = channel; + } + if (to !== undefined) { + deliveryContext.to = to; + } + if (accountId !== undefined) { + deliveryContext.accountId = accountId; + } + } + + let continuation: RestartSentinelContinuation | null | undefined; + if (value.continuation !== undefined) { + continuation = + value.continuation === null ? null : parseRestartSentinelContinuation(value.continuation); + if (continuation === null && value.continuation !== null) { + return null; + } + } + + let stats: RestartSentinelStats | null | undefined; + if (value.stats !== undefined) { + stats = value.stats === null ? null : parseRestartSentinelStats(value.stats); + if (stats === null && value.stats !== null) { + return null; + } + } + + const result: RestartSentinelPayload = { + kind: value.kind as RestartSentinelPayload["kind"], + status: value.status as RestartSentinelPayload["status"], + ts: value.ts, + }; + if (sessionKey !== undefined) { + result.sessionKey = sessionKey; + } + // SQL NULL is canonical absence for optional top-level columns. Normalize + // legacy nulls and empty routes so writes and typed-column reads agree. + if (deliveryContext !== undefined && Object.keys(deliveryContext).length > 0) { + result.deliveryContext = deliveryContext; + } + if (threadId !== undefined) { + result.threadId = threadId; + } + if (message !== undefined && message !== null) { + result.message = message; + } + if (continuation !== undefined && continuation !== null) { + result.continuation = continuation; + } + if (doctorHint !== undefined && doctorHint !== null) { + result.doctorHint = doctorHint; + } + if (stats !== undefined && stats !== null) { + result.stats = stats; + } + return result; +} + +export function parseRestartSentinelEnvelope(value: unknown): RestartSentinelEnvelope | null { + if (!isPlainRecord(value) || value.version !== 1) { + return null; + } + const payload = parseRestartSentinelPayload(value.payload); + return payload ? { version: 1, payload } : null; +} + +function parseRequiredJson(value: string | null): unknown { + if (value === null) { + return undefined; + } + try { + return JSON.parse(value) as unknown; + } catch { + return undefined; + } +} + +function decodeRestartSentinelRow(row: { + version: number; + kind: string; + status: string; + ts: number; + session_key: string | null; + thread_id: string | null; + delivery_channel: string | null; + delivery_to: string | null; + delivery_account_id: string | null; + message: string | null; + continuation_json: string | null; + doctor_hint: string | null; + stats_json: string | null; + updated_at_ms: number; +}): RestartSentinel | null { + if (row.version !== 1 || !isSafeInteger(row.updated_at_ms)) { + return null; + } + const candidate: Record = { + kind: row.kind, + status: row.status, + ts: row.ts, + }; + if (row.session_key !== null) { + candidate.sessionKey = row.session_key; + } + if (row.thread_id !== null) { + candidate.threadId = row.thread_id; + } + if ( + row.delivery_channel !== null || + row.delivery_to !== null || + row.delivery_account_id !== null + ) { + candidate.deliveryContext = { + ...(row.delivery_channel === null ? {} : { channel: row.delivery_channel }), + ...(row.delivery_to === null ? {} : { to: row.delivery_to }), + ...(row.delivery_account_id === null ? {} : { accountId: row.delivery_account_id }), + }; + } + if (row.message !== null) { + candidate.message = row.message; + } + if (row.continuation_json !== null) { + const continuation = parseRequiredJson(row.continuation_json); + if (continuation === undefined) { + return null; + } + candidate.continuation = continuation; + } + if (row.doctor_hint !== null) { + candidate.doctorHint = row.doctor_hint; + } + if (row.stats_json !== null) { + const stats = parseRequiredJson(row.stats_json); + if (stats === undefined) { + return null; + } + candidate.stats = stats; + } + const payload = parseRestartSentinelPayload(candidate); + return payload ? { version: 1, payload, revision: row.updated_at_ms } : null; +} + +export function readRestartSentinelRowSync(db: DatabaseSync): RestartSentinelRowState { + const stateDb = getNodeSqliteKysely(db); + const row = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("gateway_restart_sentinel") + .select([ + "version", + "kind", + "status", + "ts", + "session_key", + "thread_id", + "delivery_channel", + "delivery_to", + "delivery_account_id", + "message", + "continuation_json", + "doctor_hint", + "stats_json", + "updated_at_ms", + ]) + .where("sentinel_key", "=", RESTART_SENTINEL_KEY), + ); + if (!row) { + return { kind: "missing" }; + } + const sentinel = decodeRestartSentinelRow(row); + return sentinel ? { kind: "valid", sentinel } : { kind: "invalid", revision: row.updated_at_ms }; +} + +function requireValidPayload(payload: RestartSentinelPayload): RestartSentinelPayload { + const parsed = parseRestartSentinelPayload(payload); + if (!parsed) { + throw new TypeError("Invalid restart sentinel payload"); + } + return parsed; +} + +function nextRevision(currentRevision: number | null): number { + if (currentRevision !== null && !Number.isSafeInteger(currentRevision)) { + throw new Error("Restart sentinel revision is outside the safe integer range"); + } + // Same-millisecond replacements still need distinct revisions, or a stale + // consumer could delete the newer singleton row after delivering the old one. + const revision = Math.max(Date.now(), currentRevision === null ? 0 : currentRevision + 1); + if (!Number.isSafeInteger(revision)) { + throw new Error("Restart sentinel revision exhausted the safe integer range"); + } + return revision; +} + +function readRestartSentinelRevisionFloorSync(db: DatabaseSync): number | null { + const stateDb = getNodeSqliteKysely(db); + const row = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("gateway_restart_sentinel") + .select("updated_at_ms") + .where("sentinel_key", "=", RESTART_SENTINEL_REVISION_FLOOR_KEY), + ); + if (!row) { + return null; + } + if (!Number.isSafeInteger(row.updated_at_ms)) { + throw new Error("Restart sentinel revision floor is outside the safe integer range"); + } + return row.updated_at_ms; +} + +function maxRevision(left: number | null, right: number | null): number | null { + if (left === null) { + return right; + } + if (right === null) { + return left; + } + return Math.max(left, right); +} + +function buildRestartSentinelRow( + payload: RestartSentinelPayload, + revision: number, + sentinelKey = RESTART_SENTINEL_KEY, +) { + return { + sentinel_key: sentinelKey, + version: 1, + kind: payload.kind, + status: payload.status, + ts: payload.ts, + session_key: payload.sessionKey ?? null, + thread_id: payload.threadId ?? null, + delivery_channel: payload.deliveryContext?.channel ?? null, + delivery_to: payload.deliveryContext?.to ?? null, + delivery_account_id: payload.deliveryContext?.accountId ?? null, + message: payload.message ?? null, + continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null, + doctor_hint: payload.doctorHint ?? null, + stats_json: payload.stats ? JSON.stringify(payload.stats) : null, + // Debug shadow only. Reads reconstruct exclusively from typed columns above. + payload_json: JSON.stringify(payload), + updated_at_ms: revision, + }; +} + +function upsertRestartSentinelRowSync( + db: DatabaseSync, + row: ReturnType, +): void { + const stateDb = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + stateDb + .insertInto("gateway_restart_sentinel") + .values(row) + .onConflict((conflict) => + conflict.column("sentinel_key").doUpdateSet({ + version: (eb) => eb.ref("excluded.version"), + kind: (eb) => eb.ref("excluded.kind"), + status: (eb) => eb.ref("excluded.status"), + ts: (eb) => eb.ref("excluded.ts"), + session_key: (eb) => eb.ref("excluded.session_key"), + thread_id: (eb) => eb.ref("excluded.thread_id"), + delivery_channel: (eb) => eb.ref("excluded.delivery_channel"), + delivery_to: (eb) => eb.ref("excluded.delivery_to"), + delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"), + message: (eb) => eb.ref("excluded.message"), + continuation_json: (eb) => eb.ref("excluded.continuation_json"), + doctor_hint: (eb) => eb.ref("excluded.doctor_hint"), + stats_json: (eb) => eb.ref("excluded.stats_json"), + payload_json: (eb) => eb.ref("excluded.payload_json"), + updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"), + }), + ), + ); +} + +function advanceRestartSentinelRevisionFloorSync(db: DatabaseSync, revision: number): void { + // `current` is deleted after durable delivery. The reserved row survives that + // clear so a later same-millisecond write cannot reuse an idempotency revision. + const payload: RestartSentinelPayload = { kind: "restart", status: "skipped", ts: revision }; + upsertRestartSentinelRowSync( + db, + buildRestartSentinelRow(payload, revision, RESTART_SENTINEL_REVISION_FLOOR_KEY), + ); +} + +export function writeRestartSentinelRowSync( + db: DatabaseSync, + rawPayload: RestartSentinelPayload, +): RestartSentinel { + const payload = requireValidPayload(rawPayload); + const current = readRestartSentinelRowSync(db); + const currentRevision = + current.kind === "missing" + ? null + : current.kind === "valid" + ? current.sentinel.revision + : current.revision; + const revision = nextRevision( + maxRevision(currentRevision, readRestartSentinelRevisionFloorSync(db)), + ); + const row = buildRestartSentinelRow(payload, revision); + upsertRestartSentinelRowSync(db, row); + advanceRestartSentinelRevisionFloorSync(db, revision); + return { version: 1, payload, revision }; +} + +export function writeRestartSentinelRowIfRevisionSync( + db: DatabaseSync, + rawPayload: RestartSentinelPayload, + expectedRevision: number, +): RestartSentinel | null { + const current = readRestartSentinelRowSync(db); + if (current.kind !== "valid" || current.sentinel.revision !== expectedRevision) { + return null; + } + const payload = requireValidPayload(rawPayload); + const revision = nextRevision( + maxRevision(expectedRevision, readRestartSentinelRevisionFloorSync(db)), + ); + const row = buildRestartSentinelRow(payload, revision); + const stateDb = getNodeSqliteKysely(db); + const result = executeSqliteQuerySync( + db, + stateDb + .updateTable("gateway_restart_sentinel") + .set(row) + .where("sentinel_key", "=", RESTART_SENTINEL_KEY) + .where("updated_at_ms", "=", expectedRevision), + ); + if (result.numAffectedRows !== 1n) { + return null; + } + advanceRestartSentinelRevisionFloorSync(db, revision); + return { version: 1, payload, revision }; +} + +export function deleteRestartSentinelRowSync(db: DatabaseSync, expectedRevision?: number): boolean { + const current = readRestartSentinelRowSync(db); + if (current.kind === "missing") { + return false; + } + const currentRevision = current.kind === "valid" ? current.sentinel.revision : current.revision; + if (expectedRevision !== undefined && currentRevision !== expectedRevision) { + return false; + } + if (!Number.isSafeInteger(currentRevision)) { + throw new Error("Restart sentinel revision is outside the safe integer range"); + } + advanceRestartSentinelRevisionFloorSync( + db, + maxRevision(currentRevision, readRestartSentinelRevisionFloorSync(db)) ?? currentRevision, + ); + + const stateDb = getNodeSqliteKysely(db); + let query = stateDb + .deleteFrom("gateway_restart_sentinel") + .where("sentinel_key", "=", RESTART_SENTINEL_KEY); + if (expectedRevision !== undefined) { + query = query.where("updated_at_ms", "=", expectedRevision); + } + if (executeSqliteQuerySync(db, query).numAffectedRows !== 1n) { + // The outer write transaction owns both rows; fail closed so its rollback + // cannot leave a floor for a current row this call did not consume. + throw new Error("Restart sentinel changed during guarded delete"); + } + return true; +} diff --git a/src/infra/restart-sentinel.test.ts b/src/infra/restart-sentinel.test.ts index c5a46c62996b..1d8b387655bb 100644 --- a/src/infra/restart-sentinel.test.ts +++ b/src/infra/restart-sentinel.test.ts @@ -45,6 +45,7 @@ import { import { buildRestartSuccessContinuation, clearRestartSentinel, + clearRestartSentinelIfRevision, finalizeUpdateRestartSentinelRunningVersion, formatDoctorNonInteractiveHint, formatRestartSentinelMessage, @@ -87,34 +88,51 @@ function readSentinelRow() { db, stateDb .selectFrom("gateway_restart_sentinel") - .select(["sentinel_key", "version", "kind", "status", "payload_json"]) + .select(["sentinel_key", "version", "kind", "status", "payload_json", "updated_at_ms"]) .where("sentinel_key", "=", "current"), ); } -function insertSentinelRow(values: { version?: number; payloadJson: string }) { +function readSentinelRevisionFloor() { + const { db } = openOpenClawStateDatabase(); + const stateDb = getNodeSqliteKysely(db); + return executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("gateway_restart_sentinel") + .select("updated_at_ms") + .where("sentinel_key", "=", "revision-floor"), + )?.updated_at_ms; +} + +function deleteSentinelRevisionFloor() { const { db } = openOpenClawStateDatabase(); const stateDb = getNodeSqliteKysely(db); executeSqliteQuerySync( db, - stateDb.insertInto("gateway_restart_sentinel").values({ - sentinel_key: "current", - version: values.version ?? 1, - kind: "update", - status: "ok", - ts: Date.now(), - session_key: null, - thread_id: null, - delivery_channel: null, - delivery_to: null, - delivery_account_id: null, - message: null, - continuation_json: null, - doctor_hint: null, - stats_json: null, - payload_json: values.payloadJson, - updated_at_ms: Date.now(), - }), + stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "revision-floor"), + ); +} + +function updateSentinelRow( + values: Partial<{ + version: number; + kind: string; + status: string; + continuation_json: string | null; + stats_json: string | null; + payload_json: string; + updated_at_ms: number; + }>, +) { + const { db } = openOpenClawStateDatabase(); + const stateDb = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + stateDb + .updateTable("gateway_restart_sentinel") + .set(values) + .where("sentinel_key", "=", "current"), ); } @@ -147,7 +165,28 @@ describe("restart sentinel", () => { }); }); - it("imports a legacy file sentinel into sqlite once", async () => { + it("canonicalizes nullable top-level fields and empty delivery context", async () => { + await withRestartSentinelStateDir(async () => { + const written = await writeRestartSentinel({ + kind: "restart", + status: "ok", + ts: 1, + deliveryContext: {}, + message: null, + continuation: null, + doctorHint: null, + stats: null, + }); + + expect(written.payload).toEqual({ kind: "restart", status: "ok", ts: 1 }); + await expect(readRestartSentinel()).resolves.toEqual(written); + expect(readSentinelRow()?.payload_json).toBe( + JSON.stringify({ kind: "restart", status: "ok", ts: 1 }), + ); + }); + }); + + it("ignores legacy files without mutating them", async () => { await withRestartSentinelStateDir(async () => { const payload = { kind: "update" as const, @@ -161,100 +200,132 @@ describe("restart sentinel", () => { }, }; const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json"); - await fs.writeFile(legacyPath, `${JSON.stringify({ version: 1, payload })}\n`, "utf-8"); - - await expect(hasRestartSentinel()).resolves.toBe(true); - expect(readSentinelRow()).toMatchObject({ - sentinel_key: "current", - version: 1, - kind: "update", - status: "skipped", - payload_json: JSON.stringify(payload), - }); - await expect(fs.access(legacyPath)).rejects.toThrow(); - await expect(readRestartSentinel()).resolves.toEqual({ version: 1, payload }); - }); - }); - - it("does not replay a legacy file superseded by a sqlite sentinel", async () => { - await withRestartSentinelStateDir(async () => { - const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json"); - await fs.writeFile( - legacyPath, - `${JSON.stringify({ - version: 1, - payload: { - kind: "update", - status: "ok", - ts: 1, - message: "stale legacy sentinel", - }, - })}\n`, - "utf-8", - ); - - await writeRestartSentinel({ - kind: "restart", - status: "ok", - ts: 2, - message: "current sqlite sentinel", - }); - await expect(fs.access(legacyPath)).rejects.toThrow(); - - await clearRestartSentinel(); + const legacyContents = `${JSON.stringify({ version: 1, payload })}\n`; + await fs.writeFile(legacyPath, legacyContents, "utf-8"); await expect(hasRestartSentinel()).resolves.toBe(false); await expect(readRestartSentinel()).resolves.toBeNull(); + await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 }); + await clearRestartSentinel(); + await expect(fs.readFile(legacyPath, "utf-8")).resolves.toBe(legacyContents); }); }); - it("drops invalid sentinel payloads", async () => { + it("reconstructs typed columns when payload_json is corrupt", async () => { await withRestartSentinelStateDir(async () => { - insertSentinelRow({ payloadJson: "not-json" }); + const payload = { + kind: "update" as const, + status: "skipped" as const, + ts: 42, + sessionKey: "agent:main:webchat:dm:user-123", + deliveryContext: { channel: "webchat", to: "user-123", accountId: "default" }, + threadId: "thread-1", + message: "typed state", + continuation: { kind: "agentTurn" as const, message: "continue" }, + doctorHint: "run doctor", + stats: { mode: "npm", reason: "pending" }, + }; + const written = await writeRestartSentinel(payload); + updateSentinelRow({ payload_json: "not-json" }); - const read = await readRestartSentinel(); - expect(read).toBeNull(); - - expect(readSentinelRow()).toBeUndefined(); + await expect(readRestartSentinel()).resolves.toEqual(written); }); }); - it("drops structurally invalid sentinel payloads", async () => { + it("leaves malformed typed rows in place and reports them as unreadable", async () => { await withRestartSentinelStateDir(async () => { - insertSentinelRow({ version: 2, payloadJson: JSON.stringify(null) }); + await writeRestartSentinel({ kind: "update", status: "ok", ts: 1 }); + updateSentinelRow({ kind: "not-a-kind", payload_json: "{}" }); await expect(readRestartSentinel()).resolves.toBeNull(); - expect(readSentinelRow()).toBeUndefined(); + await expect(hasRestartSentinel()).resolves.toBe(false); + expect(readSentinelRow()).toMatchObject({ kind: "not-a-kind", payload_json: "{}" }); + expect(mockWarn).toHaveBeenCalledWith("Ignoring invalid typed restart sentinel row"); }); }); - it("keeps old config restart sentinels readable without restart-required stats", async () => { + it("rejects malformed typed JSON columns even when the shadow payload is valid", async () => { await withRestartSentinelStateDir(async () => { - const filePath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json"); - const payload = { - kind: "config-patch" as const, - status: "ok" as const, - ts: Date.now(), - message: "Config updated successfully", - stats: { mode: "config.patch" }, - }; - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, JSON.stringify({ version: 1, payload }, null, 2), "utf-8"); + const payload = { kind: "update" as const, status: "ok" as const, ts: 1 }; + await writeRestartSentinel(payload); + updateSentinelRow({ + continuation_json: JSON.stringify({ kind: "agentTurn", message: 42 }), + payload_json: JSON.stringify(payload), + }); - const read = await readRestartSentinel(); + await expect(readRestartSentinel()).resolves.toBeNull(); + await expect(hasRestartSentinel()).resolves.toBe(false); + }); + }); - expect(read?.payload).toEqual(payload); - if (!read) { - throw new Error("Expected old restart sentinel to be readable"); + it("keeps revisions strictly monotonic within the same millisecond", async () => { + await withRestartSentinelStateDir(async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1000); + try { + const first = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 }); + const second = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 }); + expect(second.revision).toBe(first.revision + 1); + expect(readSentinelRow()?.updated_at_ms).toBe(second.revision); + } finally { + now.mockRestore(); } - expect(summarizeRestartSentinel(read.payload)).toBe( - "Gateway restart config-patch ok (config.patch)", - ); - expect(formatRestartSentinelMessage(read.payload)).toBe( - ["Gateway restart config-patch ok (config.patch)", "Config updated successfully"].join( - "\n", - ), - ); + }); + }); + + it("upgrades pre-floor rows before unconditional and guarded clears", async () => { + await withRestartSentinelStateDir(async () => { + const now = vi.spyOn(Date, "now").mockReturnValue(1000); + try { + const first = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 }); + deleteSentinelRevisionFloor(); + expect(readSentinelRevisionFloor()).toBeUndefined(); + await expect(clearRestartSentinel()).resolves.toBe(true); + + await expect(readRestartSentinel()).resolves.toBeNull(); + await expect(hasRestartSentinel()).resolves.toBe(false); + expect(readSentinelRevisionFloor()).toBe(first.revision); + + now.mockReturnValue(500); + const second = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 }); + expect(second.revision).toBe(first.revision + 1); + + deleteSentinelRevisionFloor(); + await expect(clearRestartSentinelIfRevision(second.revision + 1)).resolves.toBe(false); + expect(readSentinelRevisionFloor()).toBeUndefined(); + await expect(readRestartSentinel()).resolves.toEqual(second); + + await expect(clearRestartSentinelIfRevision(second.revision)).resolves.toBe(true); + expect(readSentinelRevisionFloor()).toBe(second.revision); + const third = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 3 }); + expect(third.revision).toBe(second.revision + 1); + + await expect(clearRestartSentinelIfRevision(third.revision)).resolves.toBe(true); + deleteSentinelRevisionFloor(); + await expect(clearRestartSentinel()).resolves.toBe(false); + expect(readSentinelRevisionFloor()).toBeUndefined(); + } finally { + now.mockRestore(); + } + }); + }); + + it("does not let stale deletes remove a newer sentinel", async () => { + await withRestartSentinelStateDir(async () => { + const first = await writeRestartSentinel({ + kind: "restart", + status: "ok", + ts: 1, + message: "old", + }); + const newer = await writeRestartSentinel({ + kind: "restart", + status: "ok", + ts: 2, + message: "new", + }); + + await expect(clearRestartSentinelIfRevision(first.revision)).resolves.toBe(false); + await expect(readRestartSentinel()).resolves.toEqual(newer); }); }); @@ -422,7 +493,7 @@ describe("restart sentinel", () => { await finalizeUpdateRestartSentinelRunningVersion("actual-version"); - await expect(readRestartSentinel()).resolves.toEqual({ + await expect(readRestartSentinel()).resolves.toMatchObject({ version: 1, payload: { kind: "update", @@ -453,7 +524,7 @@ describe("restart sentinel", () => { await expect( finalizeUpdateRestartSentinelRunningVersion("actual-version"), ).resolves.toBeNull(); - await expect(readRestartSentinel()).resolves.toEqual({ + await expect(readRestartSentinel()).resolves.toMatchObject({ version: 1, payload: { kind: "update", @@ -481,7 +552,7 @@ describe("restart sentinel", () => { await markUpdateRestartSentinelFailure("restart-unhealthy"); - await expect(readRestartSentinel()).resolves.toEqual({ + await expect(readRestartSentinel()).resolves.toMatchObject({ version: 1, payload: { kind: "update", @@ -497,18 +568,16 @@ describe("restart sentinel", () => { }); describe("restart sentinel error visibility", () => { - it("logs a warning when clearRestartSentinel DB write fails", async () => { - mockThrowWrite.mockImplementationOnce(() => { - throw new Error("SQLITE_IOERR: disk I/O error"); - }); - + it("throws when clearRestartSentinel cannot durably delete the row", async () => { await withRestartSentinelStateDir(async () => { - await expect(clearRestartSentinel()).resolves.toBeUndefined(); + const written = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 }); + mockThrowWrite.mockImplementationOnce(() => { + throw new Error("SQLITE_IOERR: disk I/O error"); + }); - expect(mockWarn).toHaveBeenCalledTimes(1); - expect(mockWarn).toHaveBeenCalledWith( - "Failed to clear restart sentinel: SQLITE_IOERR: disk I/O error", - ); + await expect(clearRestartSentinel()).rejects.toThrow("SQLITE_IOERR: disk I/O error"); + expect(mockWarn).not.toHaveBeenCalled(); + await expect(readRestartSentinel()).resolves.toEqual(written); }); }); @@ -541,20 +610,6 @@ describe("restart sentinel error visibility", () => { ); }); }); - - it("logs a warning when the legacy sentinel path cannot be removed", async () => { - await withRestartSentinelStateDir(async () => { - const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json"); - await fs.mkdir(legacyPath); - - await expect(clearRestartSentinel()).resolves.toBeUndefined(); - - expect(mockWarn).toHaveBeenCalledTimes(1); - expect(mockWarn).toHaveBeenCalledWith( - expect.stringMatching(/^Failed to remove legacy restart sentinel: .+/), - ); - }); - }); }); describe("restart success continuation", () => { diff --git a/src/infra/restart-sentinel.ts b/src/infra/restart-sentinel.ts index 5cf0c20a0b87..4dec8810ab37 100644 --- a/src/infra/restart-sentinel.ts +++ b/src/infra/restart-sentinel.ts @@ -1,12 +1,8 @@ // Persists restart sentinel state that coordinates deferred restarts. -import { readFile, rm } from "node:fs/promises"; -import path from "node:path"; import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce"; import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { formatCliCommand } from "../cli/command-format.js"; -import { resolveStateDir } from "../config/paths.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { openOpenClawStateDatabase, runOpenClawStateWriteTransaction, @@ -14,77 +10,22 @@ import { import { resolveRuntimeServiceVersion } from "../version.js"; import { formatErrorMessage } from "./errors.js"; import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; + deleteRestartSentinelRowSync, + readRestartSentinelRowSync, + writeRestartSentinelRowIfRevisionSync, + writeRestartSentinelRowSync, + type RestartSentinel, + type RestartSentinelContinuation, + type RestartSentinelPayload, +} from "./restart-sentinel-store.js"; + +export type { + RestartSentinelContinuation, + RestartSentinelPayload, +} from "./restart-sentinel-store.js"; const sentinelLog = createSubsystemLogger("restart-sentinel"); -type RestartSentinelLog = { - stdoutTail?: string | null; - stderrTail?: string | null; - exitCode?: number | null; -}; - -type RestartSentinelStep = { - name: string; - command: string; - cwd?: string | null; - durationMs?: number | null; - log?: RestartSentinelLog | null; -}; - -type RestartSentinelStats = { - mode?: string; - root?: string; - requiresRestart?: boolean; - handoffId?: string; - before?: Record | null; - after?: Record | null; - steps?: RestartSentinelStep[]; - reason?: string | null; - durationMs?: number | null; -}; - -export type RestartSentinelContinuation = - | { - kind: "systemEvent"; - text: string; - } - | { - kind: "agentTurn"; - message: string; - }; - -export type RestartSentinelPayload = { - kind: "config-apply" | "config-auto-recovery" | "config-patch" | "update" | "restart"; - status: "ok" | "error" | "skipped"; - ts: number; - sessionKey?: string; - /** Delivery context captured at restart time to ensure channel routing survives restart. */ - deliveryContext?: { - channel?: string; - to?: string; - accountId?: string; - }; - /** Thread ID for reply threading (e.g., Slack thread_ts). */ - threadId?: string; - message?: string | null; - continuation?: RestartSentinelContinuation | null; - doctorHint?: string | null; - stats?: RestartSentinelStats | null; -}; - -type RestartSentinel = { - version: 1; - payload: RestartSentinelPayload; -}; - -const RESTART_SENTINEL_KEY = "current"; -const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json"; -type GatewayRestartSentinelDatabase = Pick; - export function formatDoctorNonInteractiveHint( env: Record = process.env as Record, ): string { @@ -97,57 +38,12 @@ export function formatDoctorNonInteractiveHint( export async function writeRestartSentinel( payload: RestartSentinelPayload, env: NodeJS.ProcessEnv = process.env, -): Promise { - const updatedAtMs = Date.now(); - runOpenClawStateWriteTransaction( - ({ db }) => { - const stateDb = getNodeSqliteKysely(db); - executeSqliteQuerySync( - db, - stateDb - .insertInto("gateway_restart_sentinel") - .values({ - sentinel_key: RESTART_SENTINEL_KEY, - version: 1, - kind: payload.kind, - status: payload.status, - ts: payload.ts, - session_key: payload.sessionKey ?? null, - thread_id: payload.threadId ?? null, - delivery_channel: payload.deliveryContext?.channel ?? null, - delivery_to: payload.deliveryContext?.to ?? null, - delivery_account_id: payload.deliveryContext?.accountId ?? null, - message: payload.message ?? null, - continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null, - doctor_hint: payload.doctorHint ?? null, - stats_json: payload.stats ? JSON.stringify(payload.stats) : null, - payload_json: JSON.stringify(payload), - updated_at_ms: updatedAtMs, - }) - .onConflict((conflict) => - conflict.column("sentinel_key").doUpdateSet({ - version: (eb) => eb.ref("excluded.version"), - kind: (eb) => eb.ref("excluded.kind"), - status: (eb) => eb.ref("excluded.status"), - ts: (eb) => eb.ref("excluded.ts"), - session_key: (eb) => eb.ref("excluded.session_key"), - thread_id: (eb) => eb.ref("excluded.thread_id"), - delivery_channel: (eb) => eb.ref("excluded.delivery_channel"), - delivery_to: (eb) => eb.ref("excluded.delivery_to"), - delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"), - message: (eb) => eb.ref("excluded.message"), - continuation_json: (eb) => eb.ref("excluded.continuation_json"), - doctor_hint: (eb) => eb.ref("excluded.doctor_hint"), - stats_json: (eb) => eb.ref("excluded.stats_json"), - payload_json: (eb) => eb.ref("excluded.payload_json"), - updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"), - }), - ), - ); - }, +): Promise { + return runOpenClawStateWriteTransaction( + ({ db }) => writeRestartSentinelRowSync(db, payload), { env }, + { operationLabel: "restart-sentinel.write" }, ); - await removeLegacyRestartSentinel(env); } function cloneRestartSentinelPayload(payload: RestartSentinelPayload): RestartSentinelPayload { @@ -158,19 +54,20 @@ async function rewriteRestartSentinel( rewrite: (payload: RestartSentinelPayload) => RestartSentinelPayload | null, env: NodeJS.ProcessEnv = process.env, ): Promise { - const current = await readRestartSentinel(env); - if (!current) { - return null; - } - const nextPayload = rewrite(cloneRestartSentinelPayload(current.payload)); - if (!nextPayload) { - return null; - } - await writeRestartSentinel(nextPayload, env); - return { - version: 1, - payload: nextPayload, - }; + return runOpenClawStateWriteTransaction( + ({ db }) => { + const current = readRestartSentinelRowSync(db); + if (current.kind !== "valid") { + return null; + } + const nextPayload = rewrite(cloneRestartSentinelPayload(current.sentinel.payload)); + return nextPayload + ? writeRestartSentinelRowIfRevisionSync(db, nextPayload, current.sentinel.revision) + : null; + }, + { env }, + { operationLabel: "restart-sentinel.rewrite-current" }, + ); } export async function finalizeUpdateRestartSentinelRunningVersion( @@ -215,60 +112,23 @@ export async function markUpdateRestartSentinelFailure( }, env); } -export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise { - try { - runOpenClawStateWriteTransaction( - ({ db }) => { - const stateDb = getNodeSqliteKysely(db); - executeSqliteQuerySync( - db, - stateDb - .deleteFrom("gateway_restart_sentinel") - .where("sentinel_key", "=", RESTART_SENTINEL_KEY), - ); - }, - { env }, - ); - } catch (err) { - // Clearing the sentinel is best-effort during shutdown/cleanup, but a - // failure here may leave the gateway believing a restart is still pending. - sentinelLog.warn(`Failed to clear restart sentinel: ${formatErrorMessage(err)}`); - } - await removeLegacyRestartSentinel(env); +export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise { + return runOpenClawStateWriteTransaction( + ({ db }) => deleteRestartSentinelRowSync(db), + { env }, + { operationLabel: "restart-sentinel.clear" }, + ); } -function resolveLegacyRestartSentinelPath(env: NodeJS.ProcessEnv): string { - return path.join(resolveStateDir(env), LEGACY_RESTART_SENTINEL_FILENAME); -} - -async function removeLegacyRestartSentinel(env: NodeJS.ProcessEnv): Promise { - try { - await rm(resolveLegacyRestartSentinelPath(env), { force: true }); - } catch (err) { - // Legacy cleanup must not block the canonical SQLite operation, but a - // failed removal can replay stale restart state after the database clears. - sentinelLog.warn(`Failed to remove legacy restart sentinel: ${formatErrorMessage(err)}`); - } -} - -async function importLegacyRestartSentinel( +export async function clearRestartSentinelIfRevision( + expectedRevision: number, env: NodeJS.ProcessEnv = process.env, -): Promise { - const legacyPath = resolveLegacyRestartSentinelPath(env); - let parsed: unknown; - try { - parsed = JSON.parse(await readFile(legacyPath, "utf-8")) as unknown; - } catch { - return null; - } - if (!isPlainRecord(parsed) || parsed.version !== 1 || !isPlainRecord(parsed.payload)) { - await removeLegacyRestartSentinel(env); - return null; - } - const payload = parsed.payload as RestartSentinelPayload; - await writeRestartSentinel(payload, env); - await removeLegacyRestartSentinel(env); - return { version: 1, payload }; +): Promise { + return runOpenClawStateWriteTransaction( + ({ db }) => deleteRestartSentinelRowSync(db, expectedRevision), + { env }, + { operationLabel: "restart-sentinel.clear-if-revision" }, + ); } export function buildRestartSuccessContinuation(params: { @@ -287,29 +147,12 @@ export async function readRestartSentinel( ): Promise { try { const database = openOpenClawStateDatabase({ env }); - const stateDb = getNodeSqliteKysely(database.db); - const row = executeSqliteQueryTakeFirstSync( - database.db, - stateDb - .selectFrom("gateway_restart_sentinel") - .select(["version", "payload_json"]) - .where("sentinel_key", "=", RESTART_SENTINEL_KEY), - ); - if (!row) { - return await importLegacyRestartSentinel(env); - } - let payload: RestartSentinelPayload | undefined; - try { - payload = JSON.parse(row.payload_json) as RestartSentinelPayload | undefined; - } catch { - await clearRestartSentinel(env); + const current = readRestartSentinelRowSync(database.db); + if (current.kind === "invalid") { + sentinelLog.warn("Ignoring invalid typed restart sentinel row"); return null; } - if (row.version !== 1 || !payload) { - await clearRestartSentinel(env); - return null; - } - return { version: 1, payload }; + return current.kind === "valid" ? current.sentinel : null; } catch (err) { sentinelLog.warn(`Failed to read restart sentinel: ${formatErrorMessage(err)}`); return null; @@ -319,18 +162,12 @@ export async function readRestartSentinel( export async function hasRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise { try { const database = openOpenClawStateDatabase({ env }); - const stateDb = getNodeSqliteKysely(database.db); - const row = executeSqliteQueryTakeFirstSync( - database.db, - stateDb - .selectFrom("gateway_restart_sentinel") - .select("sentinel_key") - .where("sentinel_key", "=", RESTART_SENTINEL_KEY), - ); - if (row) { - return true; + const current = readRestartSentinelRowSync(database.db); + if (current.kind === "invalid") { + sentinelLog.warn("Ignoring invalid typed restart sentinel row"); + return false; } - return Boolean(await importLegacyRestartSentinel(env)); + return current.kind === "valid"; } catch (err) { sentinelLog.warn(`Failed to check restart sentinel: ${formatErrorMessage(err)}`); return false; diff --git a/src/infra/session-delivery-queue-storage.ts b/src/infra/session-delivery-queue-storage.ts index b1f9df560f8f..9f1e1500ad58 100644 --- a/src/infra/session-delivery-queue-storage.ts +++ b/src/infra/session-delivery-queue-storage.ts @@ -11,6 +11,7 @@ import { moveDeliveryQueueEntryToFailed, updateDeliveryQueueEntry, upsertDeliveryQueueEntry, + type DeliveryQueueCompletionRetention, type DeliveryQueueRowMetadata, } from "./delivery-queue-sqlite.js"; import { generateSecureUuid } from "./secure-random.js"; @@ -28,6 +29,8 @@ type SessionDeliveryContext = { type SessionDeliveryRetryPolicy = { maxRetries?: number; + /** Retain terminal ownership when the durable producer can replay forever. */ + completionRetention?: DeliveryQueueCompletionRetention; }; export type SessionDeliveryRoute = { @@ -139,7 +142,9 @@ export async function enqueueSessionDelivery( entry, metadata: queuedSessionDeliveryMetadata(entry), stateDir, - reviveFailedOrCorruptPending: Boolean(params.idempotencyKey), + ...(params.completionRetention === "permanent" + ? { insertOnly: true } + : { reviveFailedOrCorruptPending: Boolean(params.idempotencyKey) }), }); return id; } diff --git a/src/infra/session-delivery-queue.storage.test.ts b/src/infra/session-delivery-queue.storage.test.ts index c0fa8fc56608..16404a8316a0 100644 --- a/src/infra/session-delivery-queue.storage.test.ts +++ b/src/infra/session-delivery-queue.storage.test.ts @@ -129,6 +129,24 @@ describe("session-delivery queue storage", () => { }); }); + it("never revives a failed permanent producer intent", async () => { + await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => { + const payload = { + kind: "systemEvent" as const, + sessionKey: "agent:main:main", + text: "restart complete", + idempotencyKey: "restart:permanent-failed", + completionRetention: "permanent" as const, + }; + const id = await enqueueSessionDelivery(payload, tempDir); + await moveSessionDeliveryToFailed(id, tempDir); + + expect(await enqueueSessionDelivery(payload, tempDir)).toBe(id); + expect(readSessionQueueStatus(tempDir, id)).toBe("failed"); + expect(await loadPendingSessionDeliveries(tempDir)).toEqual([]); + }); + }); + it("reports a completed conflict after acknowledgement", async () => { await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => { const payload = { @@ -355,4 +373,22 @@ describe("session-delivery queue storage", () => { expect(readSessionQueueStatus(tempDir, id)).toBe("completed"); }); }); + + it("retains a permanent completion receipt", async () => { + await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => { + const payload = { + kind: "systemEvent" as const, + sessionKey: "agent:main:main", + text: "restart complete", + idempotencyKey: "restart:permanent-completed", + completionRetention: "permanent" as const, + }; + const id = await enqueueSessionDelivery(payload, tempDir); + await settleSessionDelivery(id, tempDir); + + expect(await enqueueSessionDelivery(payload, tempDir)).toBe(id); + expect(readSessionQueueStatus(tempDir, id)).toBe("completed"); + expect(await loadPendingSessionDeliveries(tempDir)).toEqual([]); + }); + }); }); diff --git a/src/infra/state-migrations.doctor.ts b/src/infra/state-migrations.doctor.ts index 9904cc59876a..09abc1fb911a 100644 --- a/src/infra/state-migrations.doctor.ts +++ b/src/infra/state-migrations.doctor.ts @@ -79,6 +79,10 @@ import { detectLegacyRescuePending, discardLegacyRescuePending, } from "./state-migrations.rescue-pending.js"; +import { + detectLegacyRestartSentinel, + migrateLegacyRestartSentinel, +} from "./state-migrations.restart-sentinel.js"; import { migrateLegacyConfigHealth, migrateLegacyCurrentConversationBindings, @@ -416,6 +420,7 @@ export async function detectLegacyStateMigrations(params: { stateDir, doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, }); + const restartSentinel = detectLegacyRestartSentinel({ stateDir }); const workspace = detectLegacyWorkspaceState({ cfg: params.cfg, stateDir, @@ -598,6 +603,9 @@ export async function detectLegacyStateMigrations(params: { if (mcpOauth.hasLegacy) { preview.push("- MCP OAuth credentials: legacy JSON → shared SQLite state"); } + if (restartSentinel.hasLegacy) { + preview.push("- Restart sentinel: legacy JSON → shared SQLite state"); + } if (workspace.hasLegacy) { preview.push("- Workspace setup and attestations: legacy files → shared SQLite state"); } @@ -701,6 +709,7 @@ export async function detectLegacyStateMigrations(params: { managedOutgoingImages, apns, mcpOauth, + restartSentinel, workspace, webPush, nodeHost, @@ -909,6 +918,11 @@ export async function runLegacyStateMigrations(params: { env, stateDir: detected.stateDir, }); + const restartSentinel = await migrateLegacyRestartSentinel({ + detected: detected.restartSentinel, + env, + stateDir: detected.stateDir, + }); const workspace = await migrateLegacyWorkspaceState({ detected: detected.workspace, env, @@ -968,6 +982,7 @@ export async function runLegacyStateMigrations(params: { managedOutgoingImages, apns, mcpOauth, + restartSentinel, workspace, webPush, nodeHost, @@ -993,6 +1008,7 @@ export async function runLegacyStateMigrations(params: { ...managedOutgoingImages.changes, ...apns.changes, ...mcpOauth.changes, + ...restartSentinel.changes, ...workspace.changes, ...webPush.changes, ...nodeHost.changes, @@ -1025,6 +1041,7 @@ export async function runLegacyStateMigrations(params: { ...managedOutgoingImages.warnings, ...apns.warnings, ...mcpOauth.warnings, + ...restartSentinel.warnings, ...workspace.warnings, ...webPush.warnings, ...nodeHost.warnings, @@ -1187,6 +1204,11 @@ export async function autoMigrateLegacyState(params: { detected: detected.currentConversationBindings, stateDir: detected.stateDir, }); + const restartSentinel = await migrateLegacyRestartSentinel({ + detected: detected.restartSentinel, + env, + stateDir: detected.stateDir, + }); const channelPairing = migrateLegacyChannelPairingState({ detected: detected.channelPairing, env: { ...env, OPENCLAW_STATE_DIR: detected.stateDir }, @@ -1214,6 +1236,7 @@ export async function autoMigrateLegacyState(params: { ...configHealth.changes, ...pluginBindingApprovals.changes, ...currentConversationBindings.changes, + ...restartSentinel.changes, ...channelPairing.changes, ...preSessionChannelPlans.changes, ...pluginPlans.changes, @@ -1234,11 +1257,19 @@ export async function autoMigrateLegacyState(params: { ...configHealth.warnings, ...pluginBindingApprovals.warnings, ...currentConversationBindings.warnings, + ...restartSentinel.warnings, ...channelPairing.warnings, ...preSessionChannelPlans.warnings, ...pluginPlans.warnings, ]; - const noticeSources = [stateDirResult, detected, pluginInstallIndex, updateCheck, pluginPlans]; + const noticeSources = [ + stateDirResult, + detected, + pluginInstallIndex, + updateCheck, + restartSentinel, + pluginPlans, + ]; const notices = mergeNotices(noticeSources); logMigrationResults(changes, warnings, notices); return { @@ -1257,6 +1288,7 @@ export async function autoMigrateLegacyState(params: { configHealth.changes.length > 0 || pluginBindingApprovals.changes.length > 0 || currentConversationBindings.changes.length > 0 || + restartSentinel.changes.length > 0 || channelPairing.changes.length > 0 || preSessionChannelPlans.changes.length > 0 || pluginPlans.changes.length > 0, @@ -1282,6 +1314,7 @@ export async function autoMigrateLegacyState(params: { !detected.configHealth.hasLegacy && !detected.pluginBindingApprovals.hasLegacy && !detected.currentConversationBindings.hasLegacy && + !detected.restartSentinel?.hasLegacy && !detected.workspace.hasLegacy && !detected.channelPairing.hasLegacy ) { @@ -1350,6 +1383,11 @@ export async function autoMigrateLegacyState(params: { detected: detected.currentConversationBindings, stateDir: detected.stateDir, }); + const restartSentinel = await migrateLegacyRestartSentinel({ + detected: detected.restartSentinel, + env, + stateDir: detected.stateDir, + }); const channelPairing = migrateLegacyChannelPairingState({ detected: detected.channelPairing, env: { ...env, OPENCLAW_STATE_DIR: detected.stateDir }, @@ -1390,6 +1428,7 @@ export async function autoMigrateLegacyState(params: { ...configHealth.changes, ...pluginBindingApprovals.changes, ...currentConversationBindings.changes, + ...restartSentinel.changes, ...channelPairing.changes, ...preSessionChannelPlans.changes, ...pluginPlans.changes, @@ -1414,6 +1453,7 @@ export async function autoMigrateLegacyState(params: { ...configHealth.warnings, ...pluginBindingApprovals.warnings, ...currentConversationBindings.warnings, + ...restartSentinel.warnings, ...channelPairing.warnings, ...preSessionChannelPlans.warnings, ...pluginPlans.warnings, @@ -1422,7 +1462,14 @@ export async function autoMigrateLegacyState(params: { ...agentDir.warnings, ...channelPlans.warnings, ]; - const noticeSources = [stateDirResult, detected, pluginInstallIndex, updateCheck, pluginPlans]; + const noticeSources = [ + stateDirResult, + detected, + pluginInstallIndex, + updateCheck, + restartSentinel, + pluginPlans, + ]; const notices = mergeNotices(noticeSources); logMigrationResults(changes, warnings, notices); diff --git a/src/infra/state-migrations.restart-sentinel.test.ts b/src/infra/state-migrations.restart-sentinel.test.ts new file mode 100644 index 000000000000..aa88a37bdff1 --- /dev/null +++ b/src/infra/state-migrations.restart-sentinel.test.ts @@ -0,0 +1,365 @@ +// Covers safe startup/Doctor import of the retired restart-sentinel JSON file. +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { acquireGatewayLock } from "./gateway-lock.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { + clearRestartSentinel, + readRestartSentinel, + writeRestartSentinel, + type RestartSentinelPayload, +} from "./restart-sentinel.js"; +import { + detectLegacyRestartSentinel, + migrateLegacyRestartSentinel, +} from "./state-migrations.restart-sentinel.js"; + +type MigrationDatabase = Pick< + OpenClawStateKyselyDatabase, + "gateway_restart_sentinel" | "migration_sources" +>; + +describe("legacy restart sentinel migration", () => { + const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { + afterEach(() => { + closeOpenClawStateDatabaseForTest(); + cleanup(); + }); + }); + + function useStateDir(): { env: NodeJS.ProcessEnv; stateDir: string } { + const stateDir = tempDirs.make("openclaw-restart-sentinel-migration-"); + return { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, stateDir }; + } + + function payload(ts = 123): RestartSentinelPayload { + return { + kind: "update", + status: "ok", + ts, + sessionKey: "agent:main:main", + deliveryContext: { channel: "test", to: "target", accountId: "default" }, + threadId: "thread-1", + message: "Update completed", + continuation: { kind: "agentTurn", message: "Continue after restart" }, + doctorHint: "Run Doctor", + stats: { + mode: "managed", + handoffId: "handoff-1", + requiresRestart: true, + before: { version: "old" }, + after: { version: "new" }, + steps: [ + { + name: "install", + command: "package-manager update", + durationMs: 10, + log: { stdoutTail: "done", stderrTail: null, exitCode: 0 }, + }, + ], + }, + }; + } + + async function writeLegacy(stateDir: string, value: unknown): Promise { + const sourcePath = path.join(stateDir, "restart-sentinel.json"); + await fsp.writeFile(sourcePath, `${JSON.stringify(value)}\n`, "utf8"); + return sourcePath; + } + + async function migrate(params: { + env: NodeJS.ProcessEnv; + stateDir: string; + beforeVerify?: () => void; + removeSource?: (sourcePath: string) => Promise | void; + }) { + return await migrateLegacyRestartSentinel({ + detected: detectLegacyRestartSentinel({ stateDir: params.stateDir }), + ...params, + }); + } + + function database(env: NodeJS.ProcessEnv) { + return openOpenClawStateDatabase({ env }).db; + } + + function receipt(env: NodeJS.ProcessEnv) { + const db = database(env); + return executeSqliteQueryTakeFirstSync( + db, + getNodeSqliteKysely(db) + .selectFrom("migration_sources") + .selectAll() + .where("migration_kind", "=", "legacy-restart-sentinel-json"), + ); + } + + it("detects both the retired source and an interrupted fixed claim", async () => { + const { stateDir } = useStateDir(); + const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() }); + expect(detectLegacyRestartSentinel({ stateDir }).hasLegacy).toBe(true); + + await fsp.rename(sourcePath, `${sourcePath}.doctor-importing`); + expect(detectLegacyRestartSentinel({ stateDir }).hasLegacy).toBe(true); + }); + + it("imports and verifies the complete legacy payload before removing the source", async () => { + const { env, stateDir } = useStateDir(); + const expected = payload(); + const sourcePath = await writeLegacy(stateDir, { version: 1, payload: expected }); + + const result = await migrate({ env, stateDir }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Imported the legacy restart sentinel into shared SQLite state.", + ]); + await expect(readRestartSentinel(env)).resolves.toMatchObject({ + version: 1, + payload: expected, + }); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(receipt(env)).toMatchObject({ + removed_source: 1, + source_record_count: 1, + status: "completed", + target_table: "gateway_restart_sentinel", + }); + }); + + it("canonicalizes legacy null fields and an empty delivery context before verification", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy(stateDir, { + version: 1, + payload: { + kind: "restart", + status: "ok", + ts: 123, + deliveryContext: {}, + message: null, + continuation: null, + doctorHint: null, + stats: null, + }, + }); + + const result = await migrate({ env, stateDir }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Imported the legacy restart sentinel into shared SQLite state.", + ]); + const migrated = await readRestartSentinel(env); + expect(migrated?.payload).toEqual({ kind: "restart", status: "ok", ts: 123 }); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(receipt(env)).toMatchObject({ + removed_source: 1, + source_record_count: 1, + status: "completed", + }); + }); + + it("preserves a valid canonical row when legacy JSON conflicts", async () => { + const { env, stateDir } = useStateDir(); + const canonical = payload(999); + await writeRestartSentinel(canonical, env); + const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(1) }); + + const result = await migrate({ env, stateDir }); + + expect(result.changes).toEqual([ + "Preserved the canonical SQLite restart sentinel and discarded conflicting legacy JSON.", + ]); + await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: canonical }); + expect(fs.existsSync(sourcePath)).toBe(false); + }); + + it("repairs an invalid canonical row from a validated legacy envelope", async () => { + const { env, stateDir } = useStateDir(); + const db = database(env); + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db).insertInto("gateway_restart_sentinel").values({ + sentinel_key: "current", + version: 99, + kind: "update", + status: "ok", + ts: 1, + session_key: null, + thread_id: null, + delivery_channel: null, + delivery_to: null, + delivery_account_id: null, + message: null, + continuation_json: null, + doctor_hint: null, + stats_json: null, + payload_json: "{}", + updated_at_ms: 1, + }), + ); + const expected = payload(456); + await writeLegacy(stateDir, { version: 1, payload: expected }); + + const result = await migrate({ env, stateDir }); + + expect(result.changes).toEqual([ + "Replaced an invalid SQLite restart sentinel with validated legacy state.", + ]); + await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: expected }); + }); + + it("records and removes malformed transient state without disclosing its contents", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy(stateDir, { + version: 1, + payload: { ...payload(), ts: "invalid", message: "secret-marker" }, + }); + + const result = await migrate({ env, stateDir }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Discarded malformed retired restart sentinel JSON without importing it.", + ]); + expect(JSON.stringify(result)).not.toContain("secret-marker"); + expect(receipt(env)?.report_json).not.toContain("secret-marker"); + await expect(readRestartSentinel(env)).resolves.toBeNull(); + expect(fs.existsSync(sourcePath)).toBe(false); + }); + + it("treats a completed receipt as authoritative if the retired file reappears", async () => { + const { env, stateDir } = useStateDir(); + await writeLegacy(stateDir, { version: 1, payload: payload(1) }); + await migrate({ env, stateDir }); + await clearRestartSentinel(env); + const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(2) }); + + const result = await migrate({ env, stateDir }); + + expect(result.changes).toEqual([ + "Discarded recreated retired restart sentinel JSON using its migration receipt.", + ]); + await expect(readRestartSentinel(env)).resolves.toBeNull(); + expect(fs.existsSync(sourcePath)).toBe(false); + }); + + it("recovers an interrupted claim and finishes the same migration owner", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() }); + await fsp.rename(sourcePath, `${sourcePath}.doctor-importing`); + + const result = await migrate({ env, stateDir }); + + expect(result.warnings).toEqual([]); + await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: payload() }); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false); + }); + + it("preserves changed source bytes and records no receipt", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(1) }); + + const result = await migrate({ + env, + stateDir, + beforeVerify: () => { + fs.writeFileSync(sourcePath, JSON.stringify({ version: 1, payload: payload(2) })); + }, + }); + + expect(result.warnings).toHaveLength(1); + expect(result.warnings[0]).toContain("changed after migration loaded it"); + expect(fs.existsSync(sourcePath)).toBe(true); + expect(receipt(env)).toBeUndefined(); + }); + + it("retains a claimed source after cleanup failure and converges on retry", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() }); + const first = await migrate({ + env, + stateDir, + removeSource: () => { + throw new Error("forced cleanup failure"); + }, + }); + + expect(first.warnings).toHaveLength(1); + expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(true); + expect(receipt(env)).toMatchObject({ removed_source: 0 }); + + const second = await migrate({ env, stateDir }); + expect(second.warnings).toEqual([]); + expect(second.changes).toEqual([ + "Discarded recreated retired restart sentinel JSON using its migration receipt.", + ]); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false); + expect(receipt(env)).toMatchObject({ removed_source: 1 }); + }); + + it("requires exclusive state ownership before claiming the retired file", async () => { + const { env, stateDir } = useStateDir(); + const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() }); + const gatewayLock = await acquireGatewayLock({ + allowInTests: true, + env, + pollIntervalMs: 10, + port: 18_791, + timeoutMs: 100, + }); + if (!gatewayLock) { + throw new Error("expected test Gateway lock"); + } + let result: Awaited>; + try { + result = await migrate({ env, stateDir }); + } finally { + await gatewayLock.release(); + } + + expect(result.warnings[0]).toContain("Gateway or another SQLite maintenance command"); + expect(fs.existsSync(sourcePath)).toBe(true); + expect(receipt(env)).toBeUndefined(); + }); + + it("rejects symlinks, hardlinks, and oversized sources without deleting them", async () => { + const cases = ["symlink", "hardlink", "oversized"] as const; + for (const sourceKind of cases) { + closeOpenClawStateDatabaseForTest(); + const { env, stateDir } = useStateDir(); + const sourcePath = path.join(stateDir, "restart-sentinel.json"); + if (sourceKind === "oversized") { + await fsp.writeFile(sourcePath, Buffer.alloc(4 * 1024 * 1024 + 1)); + } else { + const targetPath = path.join(stateDir, `${sourceKind}-target.json`); + await fsp.writeFile(targetPath, JSON.stringify({ version: 1, payload: payload() })); + if (sourceKind === "symlink") { + await fsp.symlink(targetPath, sourcePath); + } else { + await fsp.link(targetPath, sourcePath); + } + } + + const result = await migrate({ env, stateDir }); + + expect(result.warnings).toHaveLength(1); + expect(fs.existsSync(sourcePath)).toBe(true); + expect(receipt(env)).toBeUndefined(); + } + }); +}); diff --git a/src/infra/state-migrations.restart-sentinel.ts b/src/infra/state-migrations.restart-sentinel.ts new file mode 100644 index 000000000000..1091e28b48b1 --- /dev/null +++ b/src/infra/state-migrations.restart-sentinel.ts @@ -0,0 +1,536 @@ +// Startup/Doctor migration for the retired restart-sentinel JSON file. +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { root, type Root } from "@openclaw/fs-safe"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; +import { formatErrorMessage } from "./errors.js"; +import { acquireGatewayLock, GatewayLockError } from "./gateway-lock.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { + parseRestartSentinelEnvelope, + readRestartSentinelRowSync, + writeRestartSentinelRowSync, + type RestartSentinelEnvelope, +} from "./restart-sentinel-store.js"; +import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js"; +import type { MigrationMessages } from "./state-migrations.types.js"; + +const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json"; +const DOCTOR_CLAIM_SUFFIX = ".doctor-importing"; +const MAX_LEGACY_RESTART_SENTINEL_BYTES = 4 * 1024 * 1024; +const MIGRATION_KIND = "legacy-restart-sentinel-json"; +const MIGRATION_LOCK_TIMEOUT_MS = 250; +const MIGRATION_LOCK_POLL_INTERVAL_MS = 25; +const utf8Decoder = new TextDecoder("utf-8", { fatal: true }); + +type RestartSentinelMigrationDatabase = Pick< + OpenClawStateKyselyDatabase, + "gateway_restart_sentinel" | "migration_runs" | "migration_sources" +>; + +type LegacySourceSnapshot = { + buffer: Buffer; + dev: number; + ino: number; + mtimeMs: number; + sha256: string; + size: number; +}; + +type MigrationDecision = + | "canonical-preserved" + | "invalid-canonical-repaired" + | "legacy-imported" + | "malformed-legacy-discarded" + | "receipt-authoritative"; + +function legacyPathMayExist(filePath: string): boolean { + try { + fs.lstatSync(filePath); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ENOENT"; + } +} + +/** Detect the exact retired file for startup preflight and explicit Doctor alike. */ +export function detectLegacyRestartSentinel(params: { + stateDir: string; +}): LegacyRestartSentinelDetection { + const sourcePath = path.join(params.stateDir, LEGACY_RESTART_SENTINEL_FILENAME); + return { + sourcePath, + hasLegacy: + legacyPathMayExist(sourcePath) || legacyPathMayExist(`${sourcePath}${DOCTOR_CLAIM_SUFFIX}`), + }; +} + +function relativeLegacyPath(stateDir: string, filePath: string): string { + const relativePath = path.relative(path.resolve(stateDir), path.resolve(filePath)); + if ( + !relativePath || + relativePath === ".." || + relativePath.startsWith(`..${path.sep}`) || + path.isAbsolute(relativePath) + ) { + throw new Error("legacy restart sentinel path is outside the state directory"); + } + return relativePath; +} + +async function readLegacySourceSnapshot( + stateRoot: Root, + stateDir: string, + sourcePath: string, +): Promise { + const opened = await stateRoot.read(relativeLegacyPath(stateDir, sourcePath), { + hardlinks: "reject", + maxBytes: MAX_LEGACY_RESTART_SENTINEL_BYTES, + symlinks: "reject", + }); + if (!opened.stat.isFile() || opened.stat.size !== opened.buffer.byteLength) { + throw new Error("legacy restart sentinel is not a stable regular file"); + } + return { + buffer: opened.buffer, + dev: opened.stat.dev, + ino: opened.stat.ino, + mtimeMs: opened.stat.mtimeMs, + sha256: createHash("sha256").update(opened.buffer).digest("hex"), + size: opened.stat.size, + }; +} + +function snapshotsMatch(left: LegacySourceSnapshot, right: LegacySourceSnapshot): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mtimeMs === right.mtimeMs && + left.sha256 === right.sha256 && + left.size === right.size + ); +} + +function parseLegacyEnvelope(snapshot: LegacySourceSnapshot): RestartSentinelEnvelope | null { + try { + return parseRestartSentinelEnvelope(JSON.parse(utf8Decoder.decode(snapshot.buffer))); + } catch { + return null; + } +} + +function receiptSourceKey(sourcePath: string): string { + return `restart-sentinel-json:${createHash("sha256").update(path.resolve(sourcePath)).digest("hex")}`; +} + +function hasMigrationReceipt(sourcePath: string, env: NodeJS.ProcessEnv): boolean { + const { db } = openOpenClawStateDatabase({ env }); + return Boolean( + executeSqliteQueryTakeFirstSync( + db, + getNodeSqliteKysely(db) + .selectFrom("migration_sources") + .select("source_key") + .where("source_key", "=", receiptSourceKey(sourcePath)), + ), + ); +} + +function decideAndRecordMigration(params: { + env: NodeJS.ProcessEnv; + sourcePath: string; + snapshot: LegacySourceSnapshot; + envelope: RestartSentinelEnvelope | null; +}): { decision: MigrationDecision; sourceKey: string } { + const sourceKey = receiptSourceKey(params.sourcePath); + const runId = `${sourceKey}:${params.snapshot.sha256.slice(0, 16)}`; + const now = Date.now(); + return runOpenClawStateWriteTransaction( + ({ db }) => { + const stateDb = getNodeSqliteKysely(db); + const receipt = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("migration_sources") + .select("source_key") + .where("source_key", "=", sourceKey), + ); + const before = readRestartSentinelRowSync(db); + let decision: MigrationDecision; + if (receipt) { + decision = "receipt-authoritative"; + } else if (!params.envelope) { + decision = "malformed-legacy-discarded"; + } else if (before.kind === "valid") { + decision = "canonical-preserved"; + } else { + const written = writeRestartSentinelRowSync(db, params.envelope.payload); + const verified = readRestartSentinelRowSync(db); + if ( + verified.kind !== "valid" || + verified.sentinel.revision !== written.revision || + !isDeepStrictEqual(verified.sentinel.payload, params.envelope.payload) + ) { + throw new Error("SQLite verification failed for the restart sentinel migration"); + } + decision = before.kind === "invalid" ? "invalid-canonical-repaired" : "legacy-imported"; + } + + const reportJson = JSON.stringify({ + source: MIGRATION_KIND, + target: "gateway_restart_sentinel", + decision, + sourceSha256: params.snapshot.sha256, + sourceValid: params.envelope !== null, + importedRecordCount: + decision === "legacy-imported" || decision === "invalid-canonical-repaired" ? 1 : 0, + preservedSqliteRecordCount: decision === "canonical-preserved" ? 1 : 0, + }); + executeSqliteQuerySync( + db, + stateDb + .insertInto("migration_runs") + .values({ + id: runId, + started_at: now, + finished_at: now, + status: "completed", + report_json: reportJson, + }) + .onConflict((conflict) => + conflict.column("id").doUpdateSet({ + finished_at: now, + status: "completed", + report_json: reportJson, + }), + ), + ); + executeSqliteQuerySync( + db, + stateDb + .insertInto("migration_sources") + .values({ + source_key: sourceKey, + migration_kind: MIGRATION_KIND, + source_path: params.sourcePath, + target_table: "gateway_restart_sentinel", + source_sha256: params.snapshot.sha256, + source_size_bytes: params.snapshot.size, + source_record_count: params.envelope ? 1 : 0, + last_run_id: runId, + status: "completed", + imported_at: now, + removed_source: 0, + report_json: reportJson, + }) + .onConflict((conflict) => + conflict.column("source_key").doUpdateSet({ + source_sha256: params.snapshot.sha256, + source_size_bytes: params.snapshot.size, + source_record_count: params.envelope ? 1 : 0, + last_run_id: runId, + status: "completed", + imported_at: now, + removed_source: 0, + report_json: reportJson, + }), + ), + ); + return { decision, sourceKey }; + }, + { env: params.env }, + ); +} + +function markSourceRemoved(sourceKey: string, env: NodeJS.ProcessEnv): void { + runOpenClawStateWriteTransaction( + ({ db }) => { + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .updateTable("migration_sources") + .set({ removed_source: 1 }) + .where("source_key", "=", sourceKey), + ); + }, + { env }, + ); +} + +async function restoreClaim(params: { + stateRoot: Root; + stateDir: string; + sourcePath: string; +}): Promise { + const claimPath = `${params.sourcePath}${DOCTOR_CLAIM_SUFFIX}`; + try { + if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, claimPath)))) { + return null; + } + if (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.sourcePath))) { + return `source path already exists: ${params.sourcePath}`; + } + await params.stateRoot.move( + relativeLegacyPath(params.stateDir, claimPath), + relativeLegacyPath(params.stateDir, params.sourcePath), + ); + return null; + } catch (error) { + return String(error); + } +} + +async function recoverInterruptedClaim(params: { + stateRoot: Root; + stateDir: string; + sourcePath: string; + env: NodeJS.ProcessEnv; +}): Promise { + const claimPath = `${params.sourcePath}${DOCTOR_CLAIM_SUFFIX}`; + const claimRelativePath = relativeLegacyPath(params.stateDir, claimPath); + if (!(await params.stateRoot.exists(claimRelativePath))) { + return; + } + if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.sourcePath)))) { + await params.stateRoot.move( + claimRelativePath, + relativeLegacyPath(params.stateDir, params.sourcePath), + ); + return; + } + // Both paths can only be retired safely when the claimed bytes already have + // an authoritative decision; otherwise preserve both for operator recovery. + if (!hasMigrationReceipt(params.sourcePath, params.env)) { + throw new Error("legacy restart sentinel source and interrupted claim both exist"); + } + await readLegacySourceSnapshot(params.stateRoot, params.stateDir, claimPath); + await params.stateRoot.remove(claimRelativePath); +} + +function decisionChange(decision: MigrationDecision): string { + switch (decision) { + case "legacy-imported": + return "Imported the legacy restart sentinel into shared SQLite state."; + case "invalid-canonical-repaired": + return "Replaced an invalid SQLite restart sentinel with validated legacy state."; + case "canonical-preserved": + return "Preserved the canonical SQLite restart sentinel and discarded conflicting legacy JSON."; + case "malformed-legacy-discarded": + return "Discarded malformed retired restart sentinel JSON without importing it."; + case "receipt-authoritative": + return "Discarded recreated retired restart sentinel JSON using its migration receipt."; + } + const unreachable: never = decision; + return unreachable; +} + +async function migrateWithExclusiveStateOwnership(params: { + detected: LegacyRestartSentinelDetection; + stateRoot: Root; + stateDir: string; + env: NodeJS.ProcessEnv; + beforeClaim?: () => void; + beforeVerify?: () => void; + removeSource?: (sourcePath: string) => Promise | void; +}): Promise { + const changes: string[] = []; + const warnings: string[] = []; + const notices: string[] = []; + const sourcePath = params.detected.sourcePath; + try { + await recoverInterruptedClaim({ + stateRoot: params.stateRoot, + stateDir: params.stateDir, + sourcePath, + env: params.env, + }); + } catch (error) { + return { + changes, + warnings: [`Failed recovering a legacy restart sentinel Doctor claim: ${String(error)}`], + }; + } + if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath)))) { + return { changes, warnings }; + } + + let snapshot: LegacySourceSnapshot; + try { + snapshot = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, sourcePath); + } catch (error) { + return { + changes, + warnings: [`Failed reading the legacy restart sentinel: ${String(error)}`], + }; + } + const envelope = parseLegacyEnvelope(snapshot); + const claimPath = `${sourcePath}${DOCTOR_CLAIM_SUFFIX}`; + try { + params.beforeVerify?.(); + const current = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, sourcePath); + if (!snapshotsMatch(current, snapshot)) { + throw new Error("legacy restart sentinel changed after migration loaded it"); + } + params.beforeClaim?.(); + await params.stateRoot.move( + relativeLegacyPath(params.stateDir, sourcePath), + relativeLegacyPath(params.stateDir, claimPath), + ); + const claimed = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, claimPath); + if (!snapshotsMatch(claimed, snapshot)) { + throw new Error("legacy restart sentinel changed before migration could claim it"); + } + } catch (error) { + const restoreError = await restoreClaim({ + stateRoot: params.stateRoot, + stateDir: params.stateDir, + sourcePath, + }); + return { + changes, + warnings: [ + `Failed claiming the legacy restart sentinel: ${String(error)}${restoreError ? `; restore failure: ${restoreError}` : ""}`, + ], + }; + } + + let result: ReturnType; + try { + result = decideAndRecordMigration({ + env: params.env, + sourcePath, + snapshot, + envelope, + }); + } catch (error) { + const restoreError = await restoreClaim({ + stateRoot: params.stateRoot, + stateDir: params.stateDir, + sourcePath, + }); + return { + changes, + warnings: [ + `Failed migrating the legacy restart sentinel: ${String(error)}${restoreError ? `; restore failure: ${restoreError}` : ""}`, + ], + }; + } + + try { + if (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath))) { + throw new Error("legacy restart sentinel reappeared during migration cleanup"); + } + if (params.removeSource) { + await params.removeSource(claimPath); + } else { + await params.stateRoot.remove(relativeLegacyPath(params.stateDir, claimPath)); + } + if ( + (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath))) || + (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, claimPath))) + ) { + throw new Error("legacy restart sentinel remains after migration cleanup"); + } + } catch (error) { + warnings.push(`Legacy restart sentinel cleanup failed: ${String(error)}`); + return { changes, warnings }; + } + + try { + markSourceRemoved(result.sourceKey, params.env); + } catch (error) { + warnings.push( + `Legacy restart sentinel was removed, but its receipt could not be finalized: ${String(error)}`, + ); + } + changes.push(decisionChange(result.decision)); + notices.push("Removed retired restart-sentinel.json after recording its migration decision."); + return { changes, warnings, notices }; +} + +/** Import or retire the old file under exclusive state ownership. */ +export async function migrateLegacyRestartSentinel(params: { + detected?: LegacyRestartSentinelDetection; + stateDir: string; + env?: NodeJS.ProcessEnv; + beforeClaim?: () => void; + beforeVerify?: () => void; + removeSource?: (sourcePath: string) => Promise | void; +}): Promise { + const detected = params.detected; + if (!detected?.hasLegacy) { + return { changes: [], warnings: [] }; + } + const env = { ...(params.env ?? process.env), OPENCLAW_STATE_DIR: params.stateDir }; + let lock: Awaited>; + try { + lock = await acquireGatewayLock({ + allowInTests: true, + env, + pollIntervalMs: MIGRATION_LOCK_POLL_INTERVAL_MS, + role: "sqlite-maintenance", + timeoutMs: MIGRATION_LOCK_TIMEOUT_MS, + }); + } catch (error) { + const detail = + error instanceof GatewayLockError + ? "the Gateway or another SQLite maintenance command owns this state directory" + : String(error); + return { + changes: [], + warnings: [ + `Failed migrating the legacy restart sentinel: ${detail}. Stop the Gateway, then run \`openclaw doctor --fix\` again.`, + ], + }; + } + if (!lock) { + return { + changes: [], + warnings: [ + "Failed migrating the legacy restart sentinel: exclusive state ownership unavailable.", + ], + }; + } + + let result: MigrationMessages = { changes: [], warnings: [] }; + let releaseError: unknown; + try { + try { + const stateRoot = await root(params.stateDir, { + hardlinks: "reject", + maxBytes: MAX_LEGACY_RESTART_SENTINEL_BYTES, + symlinks: "reject", + }); + result = await migrateWithExclusiveStateOwnership({ + ...params, + detected, + env, + stateRoot, + }); + } catch (error) { + result.warnings.push(`Failed reading the legacy restart sentinel: ${String(error)}`); + } + } finally { + try { + await lock.release(); + } catch (error) { + releaseError = error; + } + } + if (releaseError) { + result.warnings.push( + `Restart sentinel migration lock release failed: ${formatErrorMessage(releaseError)}`, + ); + } + return result; +} diff --git a/src/infra/state-migrations.restart-sentinel.types.ts b/src/infra/state-migrations.restart-sentinel.types.ts new file mode 100644 index 000000000000..6dc0635d0bcc --- /dev/null +++ b/src/infra/state-migrations.restart-sentinel.types.ts @@ -0,0 +1,4 @@ +export type LegacyRestartSentinelDetection = { + sourcePath: string; + hasLegacy: boolean; +}; diff --git a/src/infra/state-migrations.test.ts b/src/infra/state-migrations.test.ts index 0804b2b4d4b6..0ea28ea9ec3b 100644 --- a/src/infra/state-migrations.test.ts +++ b/src/infra/state-migrations.test.ts @@ -28,6 +28,7 @@ import { listWebPushSubscriptions, readPersistedVapidKeyPair, } from "./push-web-store.js"; +import { readRestartSentinel } from "./restart-sentinel.js"; import { autoMigrateLegacyState, autoMigrateLegacyPluginDoctorState, @@ -1729,7 +1730,7 @@ describe("state migrations", () => { await expect(fs.readFile(`${routingPath}.migrated`, "utf8")).resolves.toContain("robot wake"); }); - it("auto-migrates standalone legacy voice wake JSON settings", async () => { + it("auto-migrates standalone legacy JSON settings", async () => { const root = await createTempDir(); const stateDir = path.join(root, ".openclaw"); const env = createEnv(stateDir); @@ -1741,14 +1742,31 @@ describe("state migrations", () => { JSON.stringify({ triggers: ["wake"] }), "utf8", ); + const expectedSentinel = { + kind: "update" as const, + status: "ok" as const, + ts: 321, + message: "Update completed", + }; + const restartSentinelPath = path.join(stateDir, "restart-sentinel.json"); + await fs.writeFile( + restartSentinelPath, + `${JSON.stringify({ version: 1, payload: expectedSentinel })}\n`, + "utf8", + ); const result = await autoMigrateLegacyState({ cfg, env, homedir: () => root }); expect(result.skipped).toBe(false); expect(result.migrated).toBe(true); expect(result.warnings).toStrictEqual([]); + expect(result.changes).toContain( + "Imported the legacy restart sentinel into shared SQLite state.", + ); await expect(loadVoiceWakeConfig(stateDir)).resolves.toMatchObject({ triggers: ["wake"] }); + await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: expectedSentinel }); await expectMissingPath(path.join(settingsDir, "voicewake.json")); + await expectMissingPath(restartSentinelPath); }); it("runs plugin doctor migrations after repairing shared state schema", async () => { diff --git a/src/infra/state-migrations.types.ts b/src/infra/state-migrations.types.ts index 20c4ef1ba0b7..bbe853f514ff 100644 --- a/src/infra/state-migrations.types.ts +++ b/src/infra/state-migrations.types.ts @@ -3,6 +3,7 @@ import type { SessionScope } from "../config/sessions/types.js"; import type { PluginDoctorStateMigration } from "../plugins/doctor-contract-registry.js"; import type { LegacyChannelPairingStateDetection } from "./state-migrations.channel-pairing.js"; import type { LegacyMcpOAuthDetection } from "./state-migrations.mcp-oauth.types.js"; +import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js"; import type { LegacyWorkspaceStateDetection } from "./state-migrations.workspace-setup.types.js"; export type LegacyRescuePendingDetection = { @@ -115,6 +116,7 @@ export type LegacyStateDetection = { hasLegacy: boolean; }; mcpOauth: LegacyMcpOAuthDetection; + restartSentinel?: LegacyRestartSentinelDetection; workspace: LegacyWorkspaceStateDetection; webPush: { subscriptionsPath: string; diff --git a/src/infra/update-managed-service-handoff.test.ts b/src/infra/update-managed-service-handoff.test.ts index 360843a2365f..c7e94c9a107f 100644 --- a/src/infra/update-managed-service-handoff.test.ts +++ b/src/infra/update-managed-service-handoff.test.ts @@ -113,10 +113,14 @@ function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): voi doctorHint?: unknown; stats?: unknown; }; + const revision = + typeof (sentinel as { revision?: unknown }).revision === "number" + ? (sentinel as { revision: number }).revision + : Date.now(); executeSqliteQuerySync( db, stateDb.insertInto("gateway_restart_sentinel").values({ - sentinel_key: "current", + sentinel_key: record.kind === "revision-floor" ? "revision-floor" : "current", version: 1, kind: typeof record.kind === "string" ? record.kind : "update", status: typeof record.status === "string" ? record.status : "skipped", @@ -136,22 +140,34 @@ function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): voi doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null, stats_json: record.stats ? JSON.stringify(record.stats) : null, payload_json: JSON.stringify(payload), - updated_at_ms: Date.now(), + updated_at_ms: revision, }), ); } -function readRestartSentinelPayload(env: NodeJS.ProcessEnv): unknown { +function replaceRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void { + const { db } = openOpenClawStateDatabase({ env }); + const stateDb = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "current"), + ); + writeRestartSentinelRow(env, sentinel); +} + +function readRestartSentinelPayload(env: NodeJS.ProcessEnv, key = "current"): unknown { const { db } = openOpenClawStateDatabase({ env }); const stateDb = getNodeSqliteKysely(db); const row = executeSqliteQueryTakeFirstSync( db, stateDb .selectFrom("gateway_restart_sentinel") - .select(["version", "payload_json"]) - .where("sentinel_key", "=", "current"), + .select(["version", "payload_json", "updated_at_ms"]) + .where("sentinel_key", "=", key), ); - return row ? { version: row.version, payload: JSON.parse(row.payload_json) } : null; + return row + ? { version: row.version, payload: JSON.parse(row.payload_json), revision: row.updated_at_ms } + : null; } async function runHelperWithExistingSentinel(params: { @@ -159,6 +175,8 @@ async function runHelperWithExistingSentinel(params: { metaHandoffId?: string; prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise | void; sentinel?: unknown; + parentExitTimeoutMs?: number; + whileHelperRunning?: (env: NodeJS.ProcessEnv) => Promise | void; }) { const { execFile } = await vi.importActual("node:child_process"); @@ -206,7 +224,7 @@ async function runHelperWithExistingSentinel(params: { { ...helperParams, parentPid: process.pid, - parentExitTimeoutMs: 1, + parentExitTimeoutMs: params.parentExitTimeoutMs ?? 1, stateDatabasePath: resolveOpenClawStateSqlitePath(env), logPath: path.join(tmpDir, "handoff.log"), sensitivePaths: [], @@ -216,7 +234,7 @@ async function runHelperWithExistingSentinel(params: { )}\n`, ); - const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + const resultPromise = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( (resolve) => { execFile(process.execPath, [helperScriptPath, helperParamsPath], (err) => { const childError = err as (NodeJS.ErrnoException & { signal?: NodeJS.Signals }) | null; @@ -227,6 +245,8 @@ async function runHelperWithExistingSentinel(params: { }); }, ); + await params.whileHelperRunning?.(env); + const result = await resultPromise; return { result, env }; } @@ -868,7 +888,7 @@ describe("managed service update handoff", () => { }); expect(result).toEqual({ code: 1, signal: null }); - expect(readRestartSentinelPayload(env)).toEqual(unrelatedSentinel); + expect(readRestartSentinelPayload(env)).toMatchObject(unrelatedSentinel); }); it("does not overwrite a newer pending update handoff sentinel", async () => { @@ -894,7 +914,74 @@ describe("managed service update handoff", () => { }); expect(result).toEqual({ code: 1, signal: null }); - expect(readRestartSentinelPayload(env)).toEqual(newerSentinel); + expect(readRestartSentinelPayload(env)).toMatchObject(newerSentinel); + }); + + it("preserves a newer sentinel written while the detached helper is active", async () => { + const oldSentinel = { + version: 1, + revision: 100, + payload: { + kind: "update", + status: "skipped", + ts: 100, + stats: { + handoffId: "old-handoff", + reason: "managed-service-handoff-started", + }, + }, + }; + const newerSentinel = { + version: 1, + revision: 200, + payload: { + kind: "restart", + status: "ok", + ts: 200, + }, + }; + const { env } = await runHelperWithExistingSentinel({ + handoffId: "old-handoff", + metaHandoffId: "old-handoff", + sentinel: oldSentinel, + parentExitTimeoutMs: 200, + whileHelperRunning: async (stateEnv) => { + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + replaceRestartSentinelRow(stateEnv, newerSentinel); + }, + }); + + expect(readRestartSentinelPayload(env)).toMatchObject({ + payload: newerSentinel.payload, + revision: 200, + }); + }); + + it("advances the durable revision floor even when it is ahead of the clock", async () => { + const futureRevision = Date.now() + 60_000; + const { env } = await runHelperWithExistingSentinel({ + handoffId: "handoff-future-revision", + metaHandoffId: "handoff-future-revision", + sentinel: { + version: 1, + revision: futureRevision, + payload: { + kind: "revision-floor", + status: "skipped", + ts: 123, + stats: { + handoffId: "handoff-future-revision", + reason: "managed-service-handoff-started", + }, + }, + }, + }); + + expect(readRestartSentinelPayload(env, "revision-floor")).toMatchObject({ + revision: futureRevision + 1, + }); }); it("sweeps stale handoff temp directories while keeping fresh handoff logs", async () => { diff --git a/src/infra/update-managed-service-handoff.ts b/src/infra/update-managed-service-handoff.ts index db7a54671651..778a0bc2662c 100644 --- a/src/infra/update-managed-service-handoff.ts +++ b/src/infra/update-managed-service-handoff.ts @@ -201,84 +201,140 @@ function hardenStateDatabaseFiles() { } } -function readRestartSentinelPayload() { - const db = openStateDatabase(); - if (!db) { +function parseJsonColumn(value) { + if (typeof value !== "string" || !value) { return null; } try { - const row = db - .prepare("SELECT version, payload_json FROM gateway_restart_sentinel WHERE sentinel_key = ?") - .get("current"); - if (!row || row.version !== 1 || typeof row.payload_json !== "string") { - return null; - } - return JSON.parse(row.payload_json); + return JSON.parse(value); } catch { return null; - } finally { - hardenStateDatabaseFiles(); - try { - db.close(); - } catch {} } } -function writeRestartSentinelPayload(payload) { - const db = openStateDatabase(); - if (!db) { - return; +function readRestartSentinelRecord(db) { + const row = db + .prepare( + [ + "SELECT version, kind, status, ts, session_key, thread_id,", + "delivery_channel, delivery_to, delivery_account_id, message, continuation_json,", + "doctor_hint, stats_json, updated_at_ms", + "FROM gateway_restart_sentinel WHERE sentinel_key = ?", + ].join(" "), + ) + .get("current"); + if ( + !row || + row.version !== 1 || + typeof row.kind !== "string" || + typeof row.status !== "string" || + typeof row.ts !== "number" || + typeof row.updated_at_ms !== "number" + ) { + return null; } - try { - const updatedAtMs = Date.now(); - db.prepare( + const payload = { + kind: row.kind, + status: row.status, + ts: row.ts, + }; + if (typeof row.session_key === "string") payload.sessionKey = row.session_key; + if (typeof row.thread_id === "string") payload.threadId = row.thread_id; + const deliveryContext = {}; + if (typeof row.delivery_channel === "string") deliveryContext.channel = row.delivery_channel; + if (typeof row.delivery_to === "string") deliveryContext.to = row.delivery_to; + if (typeof row.delivery_account_id === "string") deliveryContext.accountId = row.delivery_account_id; + if (Object.keys(deliveryContext).length > 0) payload.deliveryContext = deliveryContext; + if (typeof row.message === "string") payload.message = row.message; + const continuation = parseJsonColumn(row.continuation_json); + if (continuation) payload.continuation = continuation; + if (typeof row.doctor_hint === "string") payload.doctorHint = row.doctor_hint; + const stats = parseJsonColumn(row.stats_json); + if (stats) payload.stats = stats; + return { revision: row.updated_at_ms, payload }; +} + +function readRestartSentinelRevisionFloor(db) { + const row = db + .prepare("SELECT updated_at_ms FROM gateway_restart_sentinel WHERE sentinel_key = ?") + .get("revision-floor"); + if (!row) return null; + if (!Number.isSafeInteger(row.updated_at_ms)) { + throw new Error("restart sentinel revision floor is outside the safe integer range"); + } + return row.updated_at_ms; +} + +function advanceRestartSentinelRevisionFloor(db, revision) { + const payloadJson = JSON.stringify({ kind: "restart", status: "skipped", ts: revision }); + db.prepare( + [ + "INSERT INTO gateway_restart_sentinel (", + "sentinel_key, version, kind, status, ts, session_key, thread_id,", + "delivery_channel, delivery_to, delivery_account_id, message, continuation_json,", + "doctor_hint, stats_json, payload_json, updated_at_ms", + ") VALUES ('revision-floor', 1, 'restart', 'skipped', ?, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ?, ?)", + "ON CONFLICT(sentinel_key) DO UPDATE SET", + "ts = excluded.ts, payload_json = excluded.payload_json, updated_at_ms = excluded.updated_at_ms", + ].join(" "), + ).run(revision, payloadJson, revision); +} + +function writeRestartSentinelPayload(db, payload, currentRevision) { + const revisionFloor = readRestartSentinelRevisionFloor(db); + const updatedAtMs = Math.max(Date.now(), Math.max(currentRevision || 0, revisionFloor || 0) + 1); + if (!Number.isSafeInteger(updatedAtMs)) { + throw new Error("restart sentinel revision exhausted the safe integer range"); + } + const values = [ + payload.kind, + payload.status, + payload.ts, + payload.sessionKey || null, + payload.threadId || null, + payload.deliveryContext && typeof payload.deliveryContext.channel === "string" + ? payload.deliveryContext.channel + : null, + payload.deliveryContext && typeof payload.deliveryContext.to === "string" + ? payload.deliveryContext.to + : null, + payload.deliveryContext && typeof payload.deliveryContext.accountId === "string" + ? payload.deliveryContext.accountId + : null, + payload.message || null, + payload.continuation ? JSON.stringify(payload.continuation) : null, + payload.doctorHint || null, + payload.stats ? JSON.stringify(payload.stats) : null, + JSON.stringify(payload), + updatedAtMs, + ]; + let changed; + if (currentRevision === null) { + changed = db.prepare( [ "INSERT INTO gateway_restart_sentinel (", "sentinel_key, version, kind, status, ts, session_key, thread_id,", "delivery_channel, delivery_to, delivery_account_id, message, continuation_json,", "doctor_hint, stats_json, payload_json, updated_at_ms", - ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - "ON CONFLICT(sentinel_key) DO UPDATE SET", - "version = excluded.version, kind = excluded.kind, status = excluded.status,", - "ts = excluded.ts, session_key = excluded.session_key, thread_id = excluded.thread_id,", - "delivery_channel = excluded.delivery_channel, delivery_to = excluded.delivery_to,", - "delivery_account_id = excluded.delivery_account_id, message = excluded.message,", - "continuation_json = excluded.continuation_json, doctor_hint = excluded.doctor_hint,", - "stats_json = excluded.stats_json, payload_json = excluded.payload_json,", - "updated_at_ms = excluded.updated_at_ms", + ") VALUES ('current', 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ].join(" "), - ).run( - "current", - 1, - payload.kind, - payload.status, - payload.ts, - payload.sessionKey || null, - payload.threadId || null, - payload.deliveryContext && typeof payload.deliveryContext.channel === "string" - ? payload.deliveryContext.channel - : null, - payload.deliveryContext && typeof payload.deliveryContext.to === "string" - ? payload.deliveryContext.to - : null, - payload.deliveryContext && typeof payload.deliveryContext.accountId === "string" - ? payload.deliveryContext.accountId - : null, - payload.message || null, - payload.continuation ? JSON.stringify(payload.continuation) : null, - payload.doctorHint || null, - payload.stats ? JSON.stringify(payload.stats) : null, - JSON.stringify(payload), - updatedAtMs, - ); - } catch (err) { - appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err))); - } finally { - hardenStateDatabaseFiles(); - try { - db.close(); - } catch {} + ).run(...values).changes === 1; + } else { + changed = db.prepare( + [ + "UPDATE gateway_restart_sentinel SET", + "version = 1, kind = ?, status = ?, ts = ?, session_key = ?, thread_id = ?,", + "delivery_channel = ?, delivery_to = ?, delivery_account_id = ?, message = ?,", + "continuation_json = ?, doctor_hint = ?, stats_json = ?, payload_json = ?, updated_at_ms = ?", + "WHERE sentinel_key = 'current' AND updated_at_ms = ?", + ].join(" "), + ).run(...values, currentRevision).changes === 1; } + if (changed) { + // This runs inside the same BEGIN IMMEDIATE section as the guarded current-row write. + advanceRestartSentinelRevisionFloor(db, updatedAtMs); + } + return changed; } function buildFallbackFailurePayload(reason) { @@ -312,22 +368,77 @@ function buildFallbackFailurePayload(reason) { } function markUpdateSentinelFailureIfPending(reason) { - let payload = readRestartSentinelPayload(); - if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) { + const snapshotDb = openStateDatabase(); + if (!snapshotDb) return; + let snapshot; + try { + snapshot = readRestartSentinelRecord(snapshotDb); + } catch { return; + } finally { + try { + snapshotDb.close(); + } catch {} } - const handoffId = typeof params.handoffId === "string" ? params.handoffId.trim() : ""; - if (payload && handoffId && (!payload.stats || payload.stats.handoffId !== handoffId)) { - return; + const fallbackPayload = snapshot === null ? buildFallbackFailurePayload(reason) : null; + + const db = openStateDatabase(); + if (!db) return; + let transactionOpen = false; + try { + db.exec("BEGIN IMMEDIATE;"); + transactionOpen = true; + const current = readRestartSentinelRecord(db); + if ( + (snapshot === null && current !== null) || + (snapshot !== null && + (current === null || current.revision !== snapshot.revision)) + ) { + db.exec("COMMIT;"); + transactionOpen = false; + return; + } + + let payload = current && current.payload; + if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) { + db.exec("COMMIT;"); + transactionOpen = false; + return; + } + const handoffId = typeof params.handoffId === "string" ? params.handoffId.trim() : ""; + if (payload && handoffId && (!payload.stats || payload.stats.handoffId !== handoffId)) { + db.exec("COMMIT;"); + transactionOpen = false; + return; + } + if (payload) { + payload = { ...payload, status: "error" }; + delete payload.continuation; + payload.stats = { ...(payload.stats || {}), reason }; + } else { + payload = fallbackPayload; + } + if (!payload) { + throw new Error("restart sentinel disappeared before guarded failure write"); + } + if (!writeRestartSentinelPayload(db, payload, current ? current.revision : null)) { + throw new Error("restart sentinel changed before guarded failure write"); + } + db.exec("COMMIT;"); + transactionOpen = false; + } catch (err) { + if (transactionOpen) { + try { + db.exec("ROLLBACK;"); + } catch {} + } + appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err))); + } finally { + hardenStateDatabaseFiles(); + try { + db.close(); + } catch {} } - if (payload) { - payload = { ...payload, status: "error" }; - delete payload.continuation; - payload.stats = { ...(payload.stats || {}), reason }; - } else { - payload = buildFallbackFailurePayload(reason); - } - writeRestartSentinelPayload(payload); } function runServiceCommand(command, args) { diff --git a/test/scripts/check-database-first-legacy-stores.test.ts b/test/scripts/check-database-first-legacy-stores.test.ts index e4ae433d204f..2818d351f4a9 100644 --- a/test/scripts/check-database-first-legacy-stores.test.ts +++ b/test/scripts/check-database-first-legacy-stores.test.ts @@ -128,6 +128,84 @@ describe("check-database-first-legacy-stores", () => { expect(violations).toEqual([{ kind: "legacy store filesystem write", line: 5 }]); }); + it("keeps legacy restart sentinel filesystem access in its sole migration owner", () => { + const runtimeViolations = collectDatabaseFirstLegacyStoreViolations( + ` + import { readFile } from "node:fs/promises"; + import path from "node:path"; + const legacyFilename = "restart-sentinel.json"; + `, + "src/infra/restart-sentinel.ts", + ); + const migrationViolations = collectDatabaseFirstLegacyStoreViolations( + ` + import { readFile } from "node:fs/promises"; + import path from "node:path"; + const legacyFilename = "restart-sentinel.json"; + `, + "src/infra/state-migrations.restart-sentinel.ts", + ); + + expect(runtimeViolations).toEqual([ + { kind: "legacy restart sentinel filesystem import", line: 2 }, + { kind: "legacy restart sentinel filesystem import", line: 3 }, + { kind: "legacy restart sentinel reference", line: 4 }, + ]); + expect(migrationViolations).toEqual([]); + }); + + it("flags legacy restart sentinel references outside the migration owner", () => { + const violations = collectDatabaseFirstLegacyStoreViolations( + ` + const legacyPath = path.join(stateDir, "restart-sentinel.json"); + await readFile(legacyPath, "utf8"); + `, + "src/commands/doctor/state-migrations.ts", + ); + + expect(violations).toEqual([{ kind: "legacy restart sentinel reference", line: 2 }]); + }); + + it("allows the CLI preflight to detect exact legacy restart sentinel inputs", () => { + const violations = collectDatabaseFirstLegacyStoreViolations( + ` + [ + path.join(stateDir, "restart-sentinel.json"), + path.join(stateDir, "restart-sentinel.json.doctor-importing"), + ].some(fileOrDirExists); + `, + "src/cli/program/config-guard.ts", + ); + + expect(violations).toEqual([]); + }); + + it("flags direct legacy restart sentinel reads from the CLI preflight", () => { + const violations = collectDatabaseFirstLegacyStoreViolations( + ` + await readFile(path.join(stateDir, "restart-sentinel.json"), "utf8"); + await readFile(path.join(stateDir, "restart-sentinel.json.doctor-importing"), "utf8"); + `, + "src/cli/program/config-guard.ts", + ); + + expect(violations).toEqual([ + { kind: "legacy restart sentinel reference", line: 2 }, + { kind: "legacy restart sentinel reference", line: 3 }, + ]); + }); + + it("flags nested restart sentinel paths disguised as CLI preflight detection", () => { + const violations = collectDatabaseFirstLegacyStoreViolations( + ` + [path.join(stateDir, "archive/restart-sentinel.json")].some(fileOrDirExists); + `, + "src/cli/program/config-guard.ts", + ); + + expect(violations).toEqual([{ kind: "legacy restart sentinel reference", line: 2 }]); + }); + it("flags retired Diffs viewer sidecar writes", () => { const violations = collectDatabaseFirstLegacyStoreViolations( ` @@ -1819,6 +1897,7 @@ describe("check-database-first-legacy-stores", () => { ); expect(violations).toEqual([ + { kind: "legacy restart sentinel reference", line: 5 }, { kind: "legacy store filesystem write", line: 5 }, { kind: "legacy store filesystem write", line: 6 }, { kind: "legacy store filesystem write", line: 7 },