refactor: store gateway restart recovery in SQLite (#110014)

* refactor: move restart sentinel state to sqlite

* fix: satisfy restart sentinel architecture gates

* test: avoid cold startup migration timeout

* fix: bound durable restart notice retries

* test: type restart notice contention mock

* fix: detect legacy restart sentinel before reads

* fix: keep delivery attempt result internal

* fix: narrow restart sentinel preflight guard
This commit is contained in:
Peter Steinberger
2026-07-17 17:43:22 +01:00
committed by GitHub
parent b75b62fb1c
commit 5366b6de81
30 changed files with 3717 additions and 633 deletions
+8
View File
@@ -159,6 +159,14 @@ SQLite before the process exits. After boot the gateway posts the outcome back
to the originating chat and dispatches a one-shot continuation turn so the
agent picks up exactly where it left off, on the same channel and thread.
The sentinel's typed SQLite columns are authoritative for restart handling;
its `payload_json` value is a replay/debug shadow only. Runtime reads, writes,
and clears SQLite state without a file fallback. During the storage cutover, a
bounded state migration runs at startup and through Doctor to preserve a
validated `restart-sentinel.json` left by the older process after an update.
The migration verifies the typed row and removes the source file before normal
restart handling continues.
## Safety valves and observability
- **Crash-loop breaker:** 3 unclean boots within 5 minutes trip a breaker that
+6 -2
View File
@@ -1256,8 +1256,12 @@ sessionId})`; create, branch, continue, list, and fork flows live in their
- Gateway restart sentinel state now uses typed shared SQLite
`gateway_restart_sentinel` rows instead of `restart-sentinel.json`; runtime
reads sentinel kind, status, routing, message, continuation, and stats from
typed columns. `payload_json` is only a replay/debug copy. Runtime code clears
the SQLite row directly and no longer carries file cleanup plumbing.
typed columns. Those columns are authoritative; `payload_json` is only a
replay/debug shadow. Runtime read, write, and clear paths are SQLite-only.
One bounded state-migration module runs during startup and Doctor to import a
validated older post-update sentinel before normal restart recovery, verify
the typed row, and remove the source file. No steady-state runtime module
reads, writes, or cleans up the legacy file.
- Gateway restart intent and supervisor handoff state now use typed shared
SQLite `gateway_restart_intent` and `gateway_restart_handoff` rows instead of
`gateway-restart-intent.json` and
+113 -5
View File
@@ -84,6 +84,26 @@ const fsSafePackageModulePattern = /^@openclaw\/fs-safe(?:\/(?:root|store))?$/u;
const bridgeMarkerPattern = /\btranscriptLocator\b|sqlite-transcript:\/\//u;
// The restart handoff must survive its one cutover migration without leaving
// filesystem fallback imports in the steady-state runtime owner.
const legacyRestartSentinelMigrationPath = "src/infra/state-migrations.restart-sentinel.ts";
const legacyRestartSentinelPreflightPath = "src/cli/program/config-guard.ts";
const legacyRestartSentinelRuntimePath = "src/infra/restart-sentinel.ts";
const legacyRestartSentinelPreflightFilenames = new Set([
"restart-sentinel.json",
"restart-sentinel.json.doctor-importing",
]);
const legacyRestartSentinelFilenamePattern =
/(?:^|[/\\])restart-sentinel\.json(?:\.doctor-importing)?$/u;
const legacyRestartSentinelRuntimeImportSpecifiers = new Set([
"fs",
"fs/promises",
"node:fs",
"node:fs/promises",
"node:path",
"path",
]);
const legacyStorePatterns = [
/\bsessions\.json\b/u,
/\.trajectory\.jsonl\b/u,
@@ -129,6 +149,7 @@ const allowedRuntimeMigrationPaths = [
"src/infra/state-migrations.managed-outgoing-images.ts",
"src/infra/state-migrations.apns.ts",
"src/infra/state-migrations.mcp-oauth.ts",
legacyRestartSentinelMigrationPath,
"src/infra/state-migrations.workspace-setup.ts",
"src/infra/state-migrations.web-push.ts",
"src/infra/state-migrations.node-host.ts",
@@ -327,6 +348,86 @@ function importSource(node) {
return ts.isStringLiteral(moduleSpecifier) ? moduleSpecifier.text : "";
}
function isLegacyRestartSentinelPreflightDetection(node, relativePath) {
if (
relativePath !== legacyRestartSentinelPreflightPath ||
!legacyRestartSentinelPreflightFilenames.has(node.text)
) {
return false;
}
const joinCall = node.parent;
if (
!ts.isCallExpression(joinCall) ||
joinCall.arguments.length !== 2 ||
joinCall.arguments[1] !== node ||
!ts.isPropertyAccessExpression(joinCall.expression) ||
!ts.isIdentifier(joinCall.expression.expression) ||
joinCall.expression.expression.text !== "path" ||
joinCall.expression.name.text !== "join" ||
!ts.isIdentifier(joinCall.arguments[0]) ||
joinCall.arguments[0].text !== "stateDir"
) {
return false;
}
const paths = joinCall.parent;
if (!ts.isArrayLiteralExpression(paths)) {
return false;
}
const someAccess = paths.parent;
if (
!ts.isPropertyAccessExpression(someAccess) ||
someAccess.expression !== paths ||
someAccess.name.text !== "some"
) {
return false;
}
const someCall = someAccess.parent;
return (
ts.isCallExpression(someCall) &&
someCall.arguments.length === 1 &&
ts.isIdentifier(someCall.arguments[0]) &&
someCall.arguments[0].text === "fileOrDirExists"
);
}
function collectLegacyRestartSentinelBoundaryViolations(sourceFile, relativePath) {
if (relativePath === legacyRestartSentinelMigrationPath) {
return [];
}
const violations = [];
const seen = new Set();
function add(node, kind) {
const line = toLine(sourceFile, node);
const key = `${line}:${kind}`;
if (seen.has(key)) {
return;
}
seen.add(key);
violations.push({ kind, line });
}
function visit(node) {
if (
ts.isStringLiteralLike(node) &&
legacyRestartSentinelFilenamePattern.test(node.text) &&
!isLegacyRestartSentinelPreflightDetection(node, relativePath)
) {
add(node, "legacy restart sentinel reference");
}
if (
relativePath === legacyRestartSentinelRuntimePath &&
ts.isImportDeclaration(node) &&
legacyRestartSentinelRuntimeImportSpecifiers.has(importSource(node))
) {
add(node, "legacy restart sentinel filesystem import");
}
ts.forEachChild(node, visit);
}
visit(sourceFile);
return violations;
}
function isHelperWriteModuleSource(source) {
return (
source === "openclaw/plugin-sdk/file-access-runtime" ||
@@ -526,21 +627,28 @@ function legacyCandidateTexts(sourceFile, node) {
*/
export function collectDatabaseFirstLegacyStoreViolations(
content,
relativePath = "source.ts",
inputRelativePath = "source.ts",
scanOptions = {},
) {
const relativePath = inputRelativePath.replaceAll("\\", "/");
const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true);
const boundaryViolations = collectLegacyRestartSentinelBoundaryViolations(
sourceFile,
relativePath,
);
if (isAllowedLegacyOwnerPath(relativePath)) {
return [];
return boundaryViolations;
}
const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true);
const currentLegacyWriteAllowances =
scanOptions.currentLegacyWriteAllowances ?? currentLegacyWriteViolationAllowances(relativePath);
const createRequireBindings = collectCreateRequireBindings(sourceFile);
const { fsModuleBindings, fsWriteAliases, fsSafeStoreFactoryAliases } =
collectFsBindings(sourceFile);
const violations = [];
const seenViolations = new Set();
const violations = [...boundaryViolations];
const seenViolations = new Set(
boundaryViolations.map((violation) => `${violation.line}:${violation.kind}`),
);
const fsModuleBindingScopes = [new Map([...fsModuleBindings].map((name) => [name, true]))];
const fsModulePropertyScopes = [new Map()];
const fsWriteAliasScopes = [fsWriteAliases];
+17
View File
@@ -207,6 +207,23 @@ describe("ensureConfigReady", () => {
});
});
it.each(["restart-sentinel.json", "restart-sentinel.json.doctor-importing"])(
"runs doctor flow when lightweight startup detection finds %s",
async (relativePath) => {
const root = useTempOpenClawHome();
writeStateMarker(root, relativePath);
await runEnsureConfigReady(["status"]);
expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({
migrateState: true,
migrateLegacyConfig: false,
invalidConfigNote: false,
observe: false,
});
},
);
it("runs doctor flow when lightweight startup detection finds a pending SQLite archive", async () => {
const root = useTempOpenClawHome();
writePendingTaskSidecarArchiveMarker(root);
+2
View File
@@ -124,6 +124,8 @@ function hasLegacyStateMigrationInputs(): boolean {
path.join(stateDir, "agent"),
path.join(stateDir, "agents"),
path.join(stateDir, "plugins", "installs.json"),
path.join(stateDir, "restart-sentinel.json"),
path.join(stateDir, "restart-sentinel.json.doctor-importing"),
path.join(stateDir, "sessions"),
path.join(stateDir, "state", "openclaw.sqlite"),
].some(fileOrDirExists) ||
@@ -0,0 +1,173 @@
// Exercises restart-notice retries against the real SQLite outbound queue.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { getDeliveryQueueEntryStatus } from "../infra/delivery-queue-sqlite.js";
import { PlatformMessageNotDispatchedError } from "../infra/outbound/deliver-types.js";
import { loadPendingDelivery } from "../infra/outbound/delivery-queue-storage.js";
import { markDeliveryPlatformSendAttemptStarted } from "../infra/outbound/delivery-queue.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
const mocks = vi.hoisted(() => ({
sendDurableMessageBatch: vi.fn(),
recoveryDeliver: vi.fn(),
resolveOutboundChannelMessageAdapter: vi.fn(() => undefined),
sleep: vi.fn(async () => {}),
}));
vi.mock("../channels/message/runtime.js", () => ({
sendDurableMessageBatch: mocks.sendDurableMessageBatch,
}));
vi.mock("../infra/outbound/deliver.js", () => ({
deliverOutboundPayloadsInternal: mocks.recoveryDeliver,
}));
vi.mock("../infra/outbound/channel-resolution.js", () => ({
resolveOutboundChannelMessageAdapter: mocks.resolveOutboundChannelMessageAdapter,
}));
vi.mock("../utils/sleep.js", () => ({ sleep: mocks.sleep }));
const { deliverRestartSentinelNotice, enqueueRestartSentinelNotice } =
await import("./server-restart-sentinel-notice.js");
type DeliveryRequest = { deliveryQueueId?: string; deliveryQueueStateDir?: string };
describe("restart sentinel notice recovery", () => {
let envSnapshot: ReturnType<typeof captureEnv> | undefined;
let stateDir = "";
const tempDirs = useAutoCleanupTempDirTracker((cleanup) => {
afterEach(() => {
closeOpenClawStateDatabaseForTest();
envSnapshot?.restore();
envSnapshot = undefined;
cleanup();
});
});
beforeEach(() => {
closeOpenClawStateDatabaseForTest();
stateDir = tempDirs.make("openclaw-restart-notice-");
envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
mocks.sendDurableMessageBatch.mockReset();
mocks.recoveryDeliver.mockReset();
mocks.resolveOutboundChannelMessageAdapter.mockClear();
mocks.sleep.mockClear();
});
async function enqueueNotice(): Promise<string> {
const queued = await enqueueRestartSentinelNotice({
channel: "whatsapp",
to: "+15550002",
message: "restart complete",
sessionKey: "agent:main:main",
revision: 123,
});
return queued.id;
}
async function deliverNotice(queueId: string): Promise<void> {
await deliverRestartSentinelNotice({
deps: {} as never,
cfg: {},
channel: "whatsapp",
to: "+15550002",
message: "restart complete",
sessionKey: "agent:main:main",
summary: "restart summary",
queueId,
});
}
async function markAttempt(request: unknown): Promise<void> {
const { deliveryQueueId, deliveryQueueStateDir } = request as DeliveryRequest;
if (!deliveryQueueId) {
throw new Error("expected durable delivery queue id");
}
await markDeliveryPlatformSendAttemptStarted(deliveryQueueId, deliveryQueueStateDir);
}
function queueStatus(queueId: string): string | undefined {
return getDeliveryQueueEntryStatus("outbound", queueId, stateDir);
}
it("replays a retryable provider-not-dispatched failure after the startup scan", async () => {
const queueId = await enqueueNotice();
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
await markAttempt(request);
return {
status: "failed",
error: new PlatformMessageNotDispatchedError("connect failed before dispatch", {
cause: new Error("connect failed"),
}),
};
});
mocks.recoveryDeliver.mockResolvedValueOnce([
{ channel: "whatsapp", messageId: "recovered-1" },
]);
await deliverNotice(queueId);
expect(mocks.recoveryDeliver).toHaveBeenCalledOnce();
expect(await loadPendingDelivery(queueId)).toBeNull();
expect(queueStatus(queueId)).toBe("completed");
});
it("does not blindly resend an ambiguous platform attempt", async () => {
const queueId = await enqueueNotice();
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
await markAttempt(request);
return { status: "failed", error: new Error("platform outcome unknown") };
});
await deliverNotice(queueId);
expect(mocks.recoveryDeliver).not.toHaveBeenCalled();
expect(await loadPendingDelivery(queueId)).toBeNull();
expect(queueStatus(queueId)).toBe("failed");
});
it("dead-letters a permanent provider rejection without replay", async () => {
const queueId = await enqueueNotice();
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
await markAttempt(request);
return {
status: "failed",
error: new PlatformMessageNotDispatchedError("payload rejected", {
cause: new Error("invalid payload"),
retryable: false,
}),
};
});
await deliverNotice(queueId);
expect(mocks.recoveryDeliver).not.toHaveBeenCalled();
expect(await loadPendingDelivery(queueId)).toBeNull();
expect(queueStatus(queueId)).toBe("failed");
});
it("preserves the shipped 45-attempt budget before dead-lettering", async () => {
const queueId = await enqueueNotice();
const retryableFailure = () =>
new PlatformMessageNotDispatchedError("transport unavailable before dispatch", {
cause: new Error("transport unavailable"),
});
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
await markAttempt(request);
return { status: "failed", error: retryableFailure() };
});
mocks.recoveryDeliver.mockImplementation(async (request) => {
await markAttempt(request);
throw retryableFailure();
});
await deliverNotice(queueId);
expect(mocks.sendDurableMessageBatch).toHaveBeenCalledOnce();
expect(mocks.recoveryDeliver).toHaveBeenCalledTimes(44);
expect(queueStatus(queueId)).toBe("failed");
});
});
@@ -0,0 +1,260 @@
// Durable outbound notice ownership for restart-sentinel recovery.
import { sendDurableMessageBatch } from "../channels/message/runtime.js";
import type { CliDeps } from "../cli/deps.types.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import {
findPlatformMessageRejectedError,
isProvenDeliveryNotSentError,
} from "../infra/delivery-recovery.shared.js";
import { formatErrorMessage } from "../infra/errors.js";
import { deliverOutboundPayloadsInternal } from "../infra/outbound/deliver.js";
import {
failPendingDelivery,
loadPendingDelivery,
reserveDeliveryAttempt,
} from "../infra/outbound/delivery-queue-storage.js";
import {
ackDelivery,
drainPendingDeliveries,
enqueueDeliveryOnce,
failDelivery,
failDeliveryAfterPlatformSend,
failDeliveryBeforePlatformSend,
withActiveDeliveryClaim,
} from "../infra/outbound/delivery-queue.js";
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
const log = createSubsystemLogger("gateway/restart-sentinel");
const RESTART_NOTICE_RECOVERY_DELAY_MS = process.env.VITEST ? 1 : 1_000;
const RESTART_NOTICE_MAX_ATTEMPTS = 45;
const RESTART_NOTICE_RECOVERY_MAX_CYCLES = RESTART_NOTICE_MAX_ATTEMPTS + 1;
type RestartSentinelNoticeRoute = {
channel: string;
to: string;
accountId?: string;
replyToId?: string;
threadId?: string;
};
export async function enqueueRestartSentinelNotice(
params: RestartSentinelNoticeRoute & {
message: string;
sessionKey: string;
revision: number;
},
): Promise<{ id: string; created: boolean }> {
return await enqueueDeliveryOnce(
{
channel: params.channel,
to: params.to,
accountId: params.accountId,
replyToId: params.replyToId,
threadId: params.threadId,
payloads: [{ text: params.message }],
bestEffort: false,
completionRetention: "permanent",
maxRetries: RESTART_NOTICE_MAX_ATTEMPTS,
},
`restart-sentinel-notice:${params.sessionKey}:${params.revision}`,
);
}
async function waitForRecoveryDrain(): Promise<void> {
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, RESTART_NOTICE_RECOVERY_DELAY_MS);
timer.unref?.();
});
}
async function drainFailedRestartSentinelNotice(params: {
cfg: OpenClawConfig;
queueId: string;
sessionKey: string;
summary: string;
}): Promise<void> {
for (let cycle = 1; cycle <= RESTART_NOTICE_RECOVERY_MAX_CYCLES; cycle += 1) {
const beforeDrain = await loadPendingDelivery(params.queueId).catch((error: unknown) => {
log.warn(`${params.summary}: restart notice recovery reload failed: ${String(error)}`, {
queueId: params.queueId,
sessionKey: params.sessionKey,
cycle,
});
return undefined;
});
if (beforeDrain === null) {
return;
}
const attemptCount = beforeDrain
? Math.max(beforeDrain.attemptCount ?? 0, beforeDrain.retryCount)
: 0;
// Atomic queue reservation blocks attempt 46. Exhausted rows get an
// immediate terminal drain; live retry attempts retain one-second spacing.
if (attemptCount < RESTART_NOTICE_MAX_ATTEMPTS) {
await waitForRecoveryDrain();
}
await drainPendingDeliveries({
drainKey: `restart-recovery:${params.queueId}`,
logLabel: `${params.summary}: restart notice recovery`,
cfg: params.cfg,
log,
deliver: deliverOutboundPayloadsInternal,
selectEntry: (entry) => ({
match: entry.id === params.queueId,
// The caller already waits between attempts. Recovery still reconciles
// send-attempt evidence before it permits recipient-visible replay.
bypassBackoff: true,
}),
}).catch((error: unknown) => {
log.warn(`${params.summary}: restart notice recovery drain failed: ${String(error)}`, {
queueId: params.queueId,
sessionKey: params.sessionKey,
cycle,
});
});
}
const pending = await loadPendingDelivery(params.queueId).catch((error: unknown) => {
log.warn(`${params.summary}: restart notice terminal reload failed: ${String(error)}`, {
queueId: params.queueId,
sessionKey: params.sessionKey,
});
return undefined;
});
if (pending === null) {
return;
}
log.warn(`${params.summary}: restart notice remains queued after bounded recovery`, {
queueId: params.queueId,
sessionKey: params.sessionKey,
retryCount: pending?.retryCount ?? null,
attemptCount: pending?.attemptCount ?? null,
maxAttempts: RESTART_NOTICE_MAX_ATTEMPTS,
});
}
export async function deliverRestartSentinelNotice(
params: RestartSentinelNoticeRoute & {
deps: CliDeps;
cfg: OpenClawConfig;
sessionKey: string;
summary: string;
message: string;
queueId: string;
},
): Promise<void> {
const claim = await withActiveDeliveryClaim(params.queueId, async () => {
try {
const reservation = await reserveDeliveryAttempt(params.queueId, RESTART_NOTICE_MAX_ATTEMPTS);
if (reservation.status === "exhausted") {
return false;
}
} catch (err) {
log.warn(
`${params.summary}: outbound delivery attempt reservation failed; queued for recovery: ${formatErrorMessage(err)}`,
{
channel: params.channel,
to: params.to,
sessionKey: params.sessionKey,
},
);
return false;
}
try {
const send = await sendDurableMessageBatch({
cfg: params.cfg,
channel: params.channel,
to: params.to,
accountId: params.accountId,
replyToId: params.replyToId,
threadId: params.threadId,
payloads: [{ text: params.message }],
session: buildOutboundSessionContext({ cfg: params.cfg, sessionKey: params.sessionKey }),
deps: params.deps,
bestEffort: false,
skipQueue: true,
deliveryQueueId: params.queueId,
});
if (send.status === "failed" || send.status === "partial_failed") {
throw send.error;
}
const results = send.status === "sent" ? send.results : [];
if (results.length === 0) {
throw new Error("outbound delivery returned no results");
}
try {
await ackDelivery(params.queueId);
return true;
} catch (err) {
const error = formatErrorMessage(err);
await failDeliveryAfterPlatformSend(params.queueId, error).catch(() => undefined);
log.warn(`${params.summary}: outbound delivery ack failed; queued for recovery: ${error}`, {
channel: params.channel,
to: params.to,
sessionKey: params.sessionKey,
});
return false;
}
} catch (err) {
// The send path records platform-attempt evidence on this queue row.
// Durable recovery owns retries so ambiguous outcomes are reconciled
// before another recipient-visible send can begin.
const error = formatErrorMessage(err);
const permanentRejection = findPlatformMessageRejectedError(err);
if (permanentRejection) {
try {
const pending = await loadPendingDelivery(params.queueId);
if (pending) {
await failPendingDelivery({
id: params.queueId,
expectedStatus: "pending",
lastError: error,
entry: pending,
});
}
} catch (persistError) {
log.warn(
`${params.summary}: permanent rejection persistence failed; queued for recovery: ${formatErrorMessage(persistError)}`,
{
channel: params.channel,
to: params.to,
sessionKey: params.sessionKey,
},
);
return false;
}
log.warn(`${params.summary}: outbound delivery permanently rejected: ${error}`, {
channel: params.channel,
to: params.to,
sessionKey: params.sessionKey,
});
return true;
}
const recordFailure = isProvenDeliveryNotSentError(err)
? failDeliveryBeforePlatformSend
: failDelivery;
await recordFailure(params.queueId, error).catch(() => undefined);
log.warn(`${params.summary}: outbound delivery failed; queued for recovery: ${String(err)}`, {
channel: params.channel,
to: params.to,
sessionKey: params.sessionKey,
});
return false;
}
});
if (claim.status === "claimed-by-other-owner") {
log.info(`${params.summary}: durable restart notice claimed by recovery`, {
sessionKey: params.sessionKey,
});
}
const needsRecovery =
claim.status === "claimed-by-other-owner" || (claim.status === "claimed" && !claim.value);
if (needsRecovery) {
await drainFailedRestartSentinelNotice({
cfg: params.cfg,
queueId: params.queueId,
sessionKey: params.sessionKey,
summary: params.summary,
});
}
}
+362 -80
View File
@@ -25,16 +25,22 @@ type AdvanceSessionDeliveryAgentRunMock =
const mocks = vi.hoisted(() => {
const state = {
queuedSessionDelivery: null as Record<string, unknown> | null,
queuedSessionDeliveries: new Map<string, Record<string, unknown>>(),
nextSessionDeliveryId: 1,
};
return {
resolveSessionAgentId: vi.fn(() => "agent-from-key"),
get queuedSessionDelivery() {
return state.queuedSessionDelivery;
return state.queuedSessionDeliveries.values().next().value ?? null;
},
set queuedSessionDelivery(value: Record<string, unknown> | null) {
state.queuedSessionDelivery = value;
state.queuedSessionDeliveries.clear();
state.nextSessionDeliveryId = 1;
if (value) {
state.queuedSessionDeliveries.set("session-delivery-1", value);
state.nextSessionDeliveryId = 2;
}
},
dispatchGatewayMethodInProcess: vi.fn<InProcessDispatchMock>(async () => ({
status: "ok",
@@ -46,6 +52,7 @@ const mocks = vi.hoisted(() => {
readRestartSentinel: vi.fn(
async (): Promise<RestartSentinel> => ({
version: 1,
revision: 123,
payload: {
kind: "restart",
status: "ok",
@@ -60,7 +67,7 @@ const mocks = vi.hoisted(() => {
}),
),
finalizeUpdateRestartSentinelRunningVersion: vi.fn(async () => null),
clearRestartSentinel: vi.fn(async () => undefined),
clearRestartSentinelIfRevision: vi.fn(async () => true),
formatRestartSentinelMessage: vi.fn(() => "restart message"),
summarizeRestartSentinel: vi.fn(() => "restart summary"),
resolveMainSessionKeyFromConfig: vi.fn(() => "agent:main:main"),
@@ -100,14 +107,34 @@ const mocks = vi.hoisted(() => {
to: "+15550002",
})) as (params?: { to?: string }) => { ok: true; to: string } | { ok: false; error: Error }),
deliverOutboundPayloads: vi.fn(async () => [{ channel: "whatsapp", messageId: "msg-1" }]),
enqueueDelivery: vi.fn(async () => "queue-1"),
enqueueDeliveryOnce: vi.fn(async (_payload: unknown, id: string) => ({ id, created: true })),
ackDelivery: vi.fn(async () => {}),
failDelivery: vi.fn(async () => {}),
failDeliveryAfterPlatformSend: vi.fn(async () => {}),
failDeliveryBeforePlatformSend: vi.fn(async () => {}),
failPendingDelivery: vi.fn(async () => ({ status: "failed" as const })),
loadPendingDelivery: vi.fn(async () => null),
drainPendingDeliveries: vi.fn(async () => {}),
reserveDeliveryAttempt: vi.fn(async () => ({
status: "reserved" as const,
attemptCount: 1,
})),
withActiveDeliveryClaim: vi.fn(async (_id: string, fn: () => Promise<unknown>) => ({
status: "claimed" as const,
value: await fn(),
})),
enqueueSystemEvent: vi.fn(),
requestHeartbeat: vi.fn(),
enqueueSessionDelivery: vi.fn(async (payload: Record<string, unknown>) => {
state.queuedSessionDelivery = payload;
return "session-delivery-1";
const existing = [...state.queuedSessionDeliveries.entries()].find(
([, entry]) => entry.idempotencyKey === payload.idempotencyKey,
);
if (existing) {
return existing[0];
}
const id = `session-delivery-${state.nextSessionDeliveryId++}`;
state.queuedSessionDeliveries.set(id, payload);
return id;
}),
ackSessionDelivery: vi.fn(async () => {}),
advanceSessionDeliveryAgentRun: vi.fn<AdvanceSessionDeliveryAgentRunMock>(async () => {}),
@@ -122,7 +149,9 @@ const mocks = vi.hoisted(() => {
messageId: "generated-media-transcript",
})),
removeCronRunContinuationSessionIfIdle: vi.fn(async () => {}),
loadPendingSessionDelivery: vi.fn(async () => state.queuedSessionDelivery),
loadPendingSessionDelivery: vi.fn(
async (id: string) => state.queuedSessionDeliveries.get(id) ?? null,
),
drainPendingSessionDeliveries: vi.fn(
async (params: {
logLabel: string;
@@ -130,7 +159,14 @@ const mocks = vi.hoisted(() => {
selectEntry: (entry: Record<string, unknown>, now: number) => { match: boolean };
deliver: (entry: Record<string, unknown>) => Promise<void>;
}) => {
if (!state.queuedSessionDelivery) {
const selected = [...state.queuedSessionDeliveries.entries()]
.map(([id, payload]) => ({ id, payload }))
.find(
({ id, payload }) =>
params.selectEntry({ id, enqueuedAt: 1, retryCount: 0, ...payload }, Date.now())
.match,
);
if (!selected) {
return;
}
const entry: Record<string, unknown> & {
@@ -138,18 +174,14 @@ const mocks = vi.hoisted(() => {
enqueuedAt: number;
retryCount: number;
} = {
id: "session-delivery-1",
id: selected.id,
enqueuedAt: 1,
retryCount: 0,
...state.queuedSessionDelivery,
...selected.payload,
};
const decision = params.selectEntry(entry, Date.now());
if (!decision.match) {
return;
}
const maxRetries = typeof entry["maxRetries"] === "number" ? entry["maxRetries"] : 5;
if (entry.retryCount >= maxRetries) {
state.queuedSessionDelivery = null;
state.queuedSessionDeliveries.delete(entry.id);
params.log.warn(
`${params.logLabel}: entry ${entry.id} exceeded max retries and was moved to failed/`,
);
@@ -157,13 +189,13 @@ const mocks = vi.hoisted(() => {
}
try {
await params.deliver(entry);
state.queuedSessionDelivery = null;
state.queuedSessionDeliveries.delete(entry.id);
} catch (err) {
state.queuedSessionDelivery = {
state.queuedSessionDeliveries.set(entry.id, {
...entry,
retryCount: entry.retryCount + 1,
lastError: err instanceof Error ? err.message : String(err),
};
});
params.log.warn(`${params.logLabel}: retry failed for entry ${entry.id}: ${String(err)}`);
}
},
@@ -215,7 +247,7 @@ vi.mock("../agents/agent-scope.js", async () => {
vi.mock("../infra/restart-sentinel.js", () => ({
finalizeUpdateRestartSentinelRunningVersion: mocks.finalizeUpdateRestartSentinelRunningVersion,
readRestartSentinel: mocks.readRestartSentinel,
clearRestartSentinel: mocks.clearRestartSentinel,
clearRestartSentinelIfRevision: mocks.clearRestartSentinelIfRevision,
formatRestartSentinelMessage: mocks.formatRestartSentinelMessage,
summarizeRestartSentinel: mocks.summarizeRestartSentinel,
}));
@@ -321,9 +353,19 @@ vi.mock("../infra/outbound/deliver.js", () => ({
}));
vi.mock("../infra/outbound/delivery-queue.js", () => ({
enqueueDelivery: mocks.enqueueDelivery,
enqueueDeliveryOnce: mocks.enqueueDeliveryOnce,
ackDelivery: mocks.ackDelivery,
failDelivery: mocks.failDelivery,
failDeliveryAfterPlatformSend: mocks.failDeliveryAfterPlatformSend,
failDeliveryBeforePlatformSend: mocks.failDeliveryBeforePlatformSend,
drainPendingDeliveries: mocks.drainPendingDeliveries,
withActiveDeliveryClaim: mocks.withActiveDeliveryClaim,
}));
vi.mock("../infra/outbound/delivery-queue-storage.js", () => ({
failPendingDelivery: mocks.failPendingDelivery,
loadPendingDelivery: mocks.loadPendingDelivery,
reserveDeliveryAttempt: mocks.reserveDeliveryAttempt,
}));
vi.mock("../infra/system-events.js", () => ({
@@ -452,6 +494,7 @@ describe("scheduleRestartSentinelWake", () => {
mocks.readRestartSentinel.mockReset();
mocks.readRestartSentinel.mockResolvedValue({
version: 1,
revision: 123,
payload: {
kind: "restart",
status: "ok",
@@ -488,10 +531,18 @@ describe("scheduleRestartSentinelWake", () => {
mocks.resolveOutboundTarget.mockReturnValue({ ok: true as const, to: "+15550002" });
mocks.deliverOutboundPayloads.mockReset();
mocks.deliverOutboundPayloads.mockResolvedValue([{ channel: "whatsapp", messageId: "msg-1" }]);
mocks.enqueueDelivery.mockReset();
mocks.enqueueDelivery.mockResolvedValue("queue-1");
mocks.enqueueDeliveryOnce.mockReset();
mocks.enqueueDeliveryOnce.mockImplementation(async (_payload, id) => ({ id, created: true }));
mocks.ackDelivery.mockClear();
mocks.failDelivery.mockClear();
mocks.failDeliveryAfterPlatformSend.mockClear();
mocks.failDeliveryBeforePlatformSend.mockClear();
mocks.failPendingDelivery.mockClear();
mocks.loadPendingDelivery.mockReset();
mocks.loadPendingDelivery.mockResolvedValue(null);
mocks.drainPendingDeliveries.mockClear();
mocks.reserveDeliveryAttempt.mockClear();
mocks.withActiveDeliveryClaim.mockClear();
mocks.enqueueSystemEvent.mockClear();
mocks.requestHeartbeat.mockClear();
mocks.enqueueSessionDelivery.mockClear();
@@ -509,7 +560,8 @@ describe("scheduleRestartSentinelWake", () => {
mocks.recoverPendingSessionDeliveries.mockClear();
mocks.finalizeUpdateRestartSentinelRunningVersion.mockReset();
mocks.finalizeUpdateRestartSentinelRunningVersion.mockResolvedValue(null);
mocks.clearRestartSentinel.mockClear();
mocks.clearRestartSentinelIfRevision.mockReset();
mocks.clearRestartSentinelIfRevision.mockResolvedValue(true);
mocks.formatRestartSentinelMessage.mockClear();
mocks.summarizeRestartSentinel.mockClear();
mocks.injectTimestamp.mockClear();
@@ -533,14 +585,21 @@ describe("scheduleRestartSentinelWake", () => {
deps,
bestEffort: false,
skipQueue: true,
deliveryQueueId: "restart-sentinel-notice:agent:main:main:123",
});
expectMockCallFields(mocks.enqueueDelivery, {
expectMockCallFields(mocks.enqueueDeliveryOnce, {
channel: "whatsapp",
to: "+15550002",
payloads: [{ text: "restart message" }],
bestEffort: false,
completionRetention: "permanent",
maxRetries: 45,
});
expect(mocks.ackDelivery).toHaveBeenCalledWith("queue-1");
expect(mocks.ackDelivery).toHaveBeenCalledWith("restart-sentinel-notice:agent:main:main:123");
expect(mocks.reserveDeliveryAttempt).toHaveBeenCalledWith(
"restart-sentinel-notice:agent:main:main:123",
45,
);
expect(mocks.failDelivery).not.toHaveBeenCalled();
expect(mocks.formatRestartSentinelMessage).toHaveBeenCalledWith(expect.anything());
expect(mocks.summarizeRestartSentinel).toHaveBeenCalledWith(expect.anything());
@@ -558,60 +617,263 @@ describe("scheduleRestartSentinelWake", () => {
expect(mocks.logWarn).not.toHaveBeenCalled();
});
it("retries outbound delivery once and logs a warning without dropping the agent wake", async () => {
vi.useFakeTimers();
mocks.deliverOutboundPayloads
.mockRejectedValueOnce(new Error("transport not ready"))
.mockResolvedValueOnce([{ channel: "whatsapp", messageId: "msg-2" }]);
it("persists every downstream intent before consuming the loaded revision", async () => {
await scheduleRestartSentinelWake({ deps: {} as never });
const wakePromise = scheduleRestartSentinelWake({ deps: {} as never });
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(1_000);
await wakePromise;
expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123);
const clearOrder = mocks.clearRestartSentinelIfRevision.mock.invocationCallOrder[0] ?? 0;
expect(mocks.enqueueSessionDelivery.mock.invocationCallOrder[0]).toBeLessThan(clearOrder);
expect(mocks.enqueueDeliveryOnce.mock.invocationCallOrder[0]).toBeLessThan(clearOrder);
expect(clearOrder).toBeLessThan(mocks.enqueueSystemEvent.mock.invocationCallOrder[0] ?? 0);
expect(clearOrder).toBeLessThan(mocks.deliverOutboundPayloads.mock.invocationCallOrder[0] ?? 0);
});
expect(mocks.enqueueDelivery).toHaveBeenCalledTimes(1);
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledTimes(2);
expectMockCallFields(mocks.deliverOutboundPayloads, { skipQueue: true }, 0);
expectMockCallFields(mocks.deliverOutboundPayloads, { skipQueue: true }, 1);
expect(mocks.ackDelivery).toHaveBeenCalledWith("queue-1");
it("stops delivery when guarded sentinel consumption fails", async () => {
mocks.clearRestartSentinelIfRevision.mockRejectedValueOnce(new Error("database locked"));
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.enqueueSessionDelivery).toHaveBeenCalledOnce();
expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledOnce();
expect(mocks.enqueueSystemEvent).not.toHaveBeenCalled();
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", {
source: "restart-sentinel",
sessionKey: "agent:main:main",
reason: "database locked",
});
});
it("preserves a newer sentinel while draining durable work from the loaded revision", async () => {
mocks.clearRestartSentinelIfRevision.mockResolvedValueOnce(false);
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123);
expect(mocks.enqueueSystemEvent).toHaveBeenCalledOnce();
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce();
expect(mocks.logInfo).toHaveBeenCalledWith(
"restart summary: newer restart sentinel preserved while draining durable work",
{ sessionKey: "agent:main:main" },
);
});
it("does not resend a restart notice whose stable queue id is already owned", async () => {
mocks.enqueueDeliveryOnce.mockImplementationOnce(async (_payload, id) => ({
id,
created: false,
}));
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123);
expect(mocks.enqueueDeliveryOnce.mock.calls[0]?.[1]).toBe(
"restart-sentinel-notice:agent:main:main:123",
);
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
expect(mocks.ackDelivery).not.toHaveBeenCalled();
expect(mocks.failDelivery).not.toHaveBeenCalled();
expect(mocks.enqueueSystemEvent).toHaveBeenCalledTimes(1);
expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1);
expect(mocks.logWarn.mock.calls).toEqual([
[
"restart summary: outbound delivery failed; retrying in 1000ms: Error: transport not ready",
{
channel: "whatsapp",
to: "+15550002",
sessionKey: "agent:main:main",
attempt: 1,
maxAttempts: 45,
},
],
expect(mocks.logInfo).toHaveBeenCalledWith(
"restart summary: durable restart notice already owned",
{ sessionKey: "agent:main:main" },
);
});
it("queues the restart wake before a system-event continuation", async () => {
mocks.readRestartSentinel.mockResolvedValueOnce({
version: 1,
revision: 123,
payload: {
kind: "restart",
status: "ok",
ts: 99,
sessionKey: "agent:main:main",
continuation: { kind: "systemEvent", text: "continue" },
},
});
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.enqueueSessionDelivery).toHaveBeenCalledTimes(2);
expect(mocks.enqueueSessionDelivery).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
text: "restart message",
idempotencyKey: "restart-sentinel-wake:agent:main:main:123",
completionRetention: "permanent",
}),
);
expect(mocks.enqueueSessionDelivery).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
text: "continue",
idempotencyKey: "restart-sentinel:agent:main:main:systemEvent:123",
completionRetention: "permanent",
}),
);
expect(mocks.enqueueSystemEvent.mock.calls.map((call) => call[0])).toEqual([
"restart message",
"continue",
]);
});
it("keeps one queued restart notice when outbound retries are exhausted", async () => {
vi.useFakeTimers();
mocks.deliverOutboundPayloads.mockRejectedValue(new Error("transport still not ready"));
it("queues a failed outbound notice for durable recovery without dropping the agent wake", async () => {
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("platform outcome unknown"));
mocks.loadPendingDelivery
.mockResolvedValueOnce({
id: "restart-sentinel-notice:agent:main:main:123",
retryCount: 1,
lastError: "platform outcome unknown",
} as never)
.mockResolvedValue(null);
const wakePromise = scheduleRestartSentinelWake({ deps: {} as never });
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(44_000);
await wakePromise;
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.enqueueDelivery).toHaveBeenCalledTimes(1);
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledTimes(45);
expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledTimes(1);
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce();
expectMockCallFields(mocks.deliverOutboundPayloads, {
skipQueue: true,
deliveryQueueId: "restart-sentinel-notice:agent:main:main:123",
});
expect(mocks.ackDelivery).not.toHaveBeenCalled();
expect(mocks.failDelivery).toHaveBeenCalledWith("queue-1", "transport still not ready");
expect(mocks.failDelivery).toHaveBeenCalledWith(
"restart-sentinel-notice:agent:main:main:123",
"platform outcome unknown",
);
expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce();
const drain = expectRecordFields(mockCallArg(mocks.drainPendingDeliveries), {
drainKey: "restart-recovery:restart-sentinel-notice:agent:main:main:123",
deliver: expect.any(Function),
});
const selectEntry = drain.selectEntry as (entry: { id: string }) => {
match: boolean;
bypassBackoff?: boolean;
};
expect(selectEntry({ id: "restart-sentinel-notice:agent:main:main:123" })).toEqual({
match: true,
bypassBackoff: true,
});
expect(selectEntry({ id: "other" })).toEqual({ match: false, bypassBackoff: true });
expect(mocks.enqueueSystemEvent).toHaveBeenCalledTimes(1);
expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1);
expect(mocks.logWarn).toHaveBeenCalledWith(
"restart summary: outbound delivery failed; queued for recovery: Error: platform outcome unknown",
{
channel: "whatsapp",
to: "+15550002",
sessionKey: "agent:main:main",
},
);
});
it("still dispatches continuation after restart notice retries are exhausted", async () => {
vi.useFakeTimers();
mocks.deliverOutboundPayloads.mockRejectedValue(new Error("transport still not ready"));
it("starts a terminal drain when the persisted retry budget is exhausted", async () => {
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport unavailable"));
mocks.loadPendingDelivery.mockResolvedValue({
id: "restart-sentinel-notice:agent:main:main:123",
retryCount: 45,
attemptCount: 45,
lastError: "transport unavailable",
} as never);
mocks.drainPendingDeliveries.mockImplementationOnce(async () => {
mocks.loadPendingDelivery.mockResolvedValue(null);
});
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce();
});
it("preserves a notice whose persisted retry budget is not exhausted", async () => {
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport unavailable"));
mocks.loadPendingDelivery.mockResolvedValue({
id: "restart-sentinel-notice:agent:main:main:123",
retryCount: 7,
lastError: "database busy",
} as never);
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.drainPendingDeliveries).toHaveBeenCalledTimes(46);
expect(mocks.failPendingDelivery).not.toHaveBeenCalled();
expect(mocks.logWarn).toHaveBeenCalledWith(
"restart summary: restart notice remains queued after bounded recovery",
{
queueId: "restart-sentinel-notice:agent:main:main:123",
sessionKey: "agent:main:main",
retryCount: 7,
attemptCount: null,
maxAttempts: 45,
},
);
});
it("continues exact-id recovery after another owner releases the notice", async () => {
mocks.withActiveDeliveryClaim.mockResolvedValueOnce({
status: "claimed-by-other-owner",
} as never);
mocks.loadPendingDelivery
.mockResolvedValueOnce({
id: "restart-sentinel-notice:agent:main:main:123",
retryCount: 0,
} as never)
.mockResolvedValue(null);
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce();
expect(mocks.logInfo).toHaveBeenCalledWith(
"restart summary: durable restart notice claimed by recovery",
{ sessionKey: "agent:main:main" },
);
});
it("schedules safe recovery when the delivered notice cannot be acknowledged", async () => {
mocks.ackDelivery.mockRejectedValueOnce(new Error("ack unavailable"));
mocks.loadPendingDelivery
.mockResolvedValueOnce({
id: "restart-sentinel-notice:agent:main:main:123",
retryCount: 1,
recoveryState: "unknown_after_send",
} as never)
.mockResolvedValue(null);
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.failDeliveryAfterPlatformSend).toHaveBeenCalledWith(
"restart-sentinel-notice:agent:main:main:123",
"ack unavailable",
);
expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce();
expect(mocks.logWarn).toHaveBeenCalledWith(
"restart summary: outbound delivery ack failed; queued for recovery: ack unavailable",
{
channel: "whatsapp",
to: "+15550002",
sessionKey: "agent:main:main",
},
);
});
it("keeps one queued restart notice when outbound delivery fails", async () => {
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport still not ready"));
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledTimes(1);
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce();
expect(mocks.ackDelivery).not.toHaveBeenCalled();
expect(mocks.failDelivery).toHaveBeenCalledWith(
"restart-sentinel-notice:agent:main:main:123",
"transport still not ready",
);
});
it("still dispatches continuation after a restart notice is queued for recovery", async () => {
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport still not ready"));
mocks.readRestartSentinel.mockResolvedValue({
version: 1,
revision: 123,
payload: {
sessionKey: "agent:main:main",
deliveryContext: {
@@ -627,13 +889,12 @@ describe("scheduleRestartSentinelWake", () => {
},
} as unknown as Awaited<ReturnType<typeof mocks.readRestartSentinel>>);
const wakePromise = scheduleRestartSentinelWake({ deps: {} as never });
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(44_000);
await wakePromise;
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.failDelivery).toHaveBeenCalledWith("queue-1", "transport still not ready");
expect(mocks.failDelivery).toHaveBeenCalledWith(
"restart-sentinel-notice:agent:main:main:123",
"transport still not ready",
);
expect(mocks.recordInboundSessionAndDispatchReply).toHaveBeenCalledTimes(1);
expectContinuationDispatchFields({ routeSessionKey: "agent:main:main" }, { Body: "continue" });
});
@@ -693,7 +954,7 @@ describe("scheduleRestartSentinelWake", () => {
await scheduleRestartSentinelWake({ deps: {} as never });
expectMockCallFields(mocks.enqueueDelivery, {
expectMockCallFields(mocks.enqueueDeliveryOnce, {
payloads: [{ text: "restart message" }],
threadId: "thread-42",
});
@@ -2114,6 +2375,8 @@ describe("scheduleRestartSentinelWake", () => {
it("dispatches agentTurn continuation for a completed run entry", async () => {
mocks.readRestartSentinel.mockResolvedValue({
version: 1,
revision: 123,
payload: {
sessionKey: "agent:main:main",
deliveryContext: {
@@ -2153,6 +2416,7 @@ describe("scheduleRestartSentinelWake", () => {
message: "continue after restart",
messageId: "restart-sentinel:agent:main:main:agentTurn:123",
expectedSessionId: "agent:main:main",
completionRetention: "permanent",
route: {
channel: "whatsapp",
to: "+15550002",
@@ -2649,6 +2913,8 @@ describe("scheduleRestartSentinelWake", () => {
const busyReply = "⚠️ Previous run is still shutting down. Please try again in a moment.";
let attempt = 0;
mocks.readRestartSentinel.mockResolvedValue({
version: 1,
revision: 123,
payload: {
sessionKey: "agent:main:main",
deliveryContext: {
@@ -2766,7 +3032,7 @@ describe("scheduleRestartSentinelWake", () => {
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.clearRestartSentinel).not.toHaveBeenCalled();
expect(mocks.clearRestartSentinelIfRevision).not.toHaveBeenCalled();
expect(mocks.drainPendingSessionDeliveries).not.toHaveBeenCalled();
expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", {
source: "restart-sentinel",
@@ -2825,12 +3091,13 @@ describe("scheduleRestartSentinelWake", () => {
};
mocks.readRestartSentinel.mockResolvedValue({
version: 1,
revision: 123,
payload,
});
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce();
expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledOnce();
expect(getLatestUpdateRestartSentinel()).toEqual(payload);
});
@@ -2847,6 +3114,7 @@ describe("scheduleRestartSentinelWake", () => {
};
mocks.readRestartSentinel.mockResolvedValue({
version: 1,
revision: 123,
payload,
});
@@ -2856,7 +3124,7 @@ describe("scheduleRestartSentinelWake", () => {
expect(getLatestUpdateRestartSentinel()).toEqual(payload);
});
it("does not wake the main session when the sentinel has no sessionKey", async () => {
it("durably wakes the main session when the sentinel has no sessionKey", async () => {
mocks.readRestartSentinel.mockResolvedValue({
payload: {
message: "restart message",
@@ -2868,7 +3136,12 @@ describe("scheduleRestartSentinelWake", () => {
expect(mocks.enqueueSystemEvent).toHaveBeenCalledWith("restart message", {
sessionKey: "agent:main:main",
});
expect(mocks.requestHeartbeat).not.toHaveBeenCalled();
expect(mocks.requestHeartbeat).toHaveBeenCalledWith({
source: "restart-sentinel",
intent: "immediate",
reason: "wake",
sessionKey: "agent:main:main",
});
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
});
@@ -2910,6 +3183,15 @@ describe("scheduleRestartSentinelWake", () => {
threadId: undefined,
});
mocks.deliveryContextFromSession.mockReturnValue(undefined);
mocks.loadSessionEntry.mockReturnValue({
cfg: {},
entry: { sessionId: "agent:main:matrix:channel:!lowercased:example.org", updatedAt: 0 },
store: {},
storePath: "/tmp/sessions.json",
canonicalKey: "agent:main:matrix:channel:!lowercased:example.org",
storeKeys: ["agent:main:matrix:channel:!lowercased:example.org"],
legacyKey: undefined,
});
await scheduleRestartSentinelWake({ deps: {} as never });
@@ -2918,7 +3200,7 @@ describe("scheduleRestartSentinelWake", () => {
sessionKey: "agent:main:matrix:channel:!lowercased:example.org",
});
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
expect(mocks.enqueueDelivery).not.toHaveBeenCalled();
expect(mocks.enqueueDeliveryOnce).not.toHaveBeenCalled();
expect(mocks.resolveOutboundTarget).not.toHaveBeenCalled();
});
+84 -98
View File
@@ -5,7 +5,6 @@ import { REPLY_RUN_STILL_SHUTTING_DOWN_TEXT } from "../auto-reply/reply/get-repl
import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
import { dispatchReplyWithBufferedBlockDispatcher } from "../auto-reply/reply/provider-dispatcher.js";
import type { ChatType } from "../channels/chat-type.js";
import { sendDurableMessageBatch } from "../channels/message/runtime.js";
import { getChannelPlugin, normalizeChannelId } from "../channels/plugins/index.js";
import { recordInboundSession } from "../channels/session.js";
import { dispatchAssembledChannelTurn } from "../channels/turn/kernel.js";
@@ -14,11 +13,9 @@ import { resolveMainSessionKeyFromConfig } from "../config/sessions.js";
import { parseSessionThreadInfo } from "../config/sessions/thread-info.js";
import { formatErrorMessage, toErrorObject } from "../infra/errors.js";
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
import { ackDelivery, enqueueDelivery, failDelivery } from "../infra/outbound/delivery-queue.js";
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
import { resolveOutboundTarget } from "../infra/outbound/targets.js";
import {
clearRestartSentinel,
clearRestartSentinelIfRevision,
finalizeUpdateRestartSentinelRunningVersion,
formatRestartSentinelMessage,
readRestartSentinel,
@@ -53,12 +50,14 @@ import {
} from "../utils/delivery-context.shared.js";
import { INTERNAL_MESSAGE_CHANNEL } from "../utils/message-channel.js";
import { deliverQueuedGeneratedMediaAgentTurn } from "./server-restart-sentinel-agent-delivery.js";
import {
deliverRestartSentinelNotice,
enqueueRestartSentinelNotice,
} from "./server-restart-sentinel-notice.js";
import { loadSessionEntry } from "./session-utils.js";
import { runStartupTasks, type StartupTask } from "./startup-tasks.js";
const log = createSubsystemLogger("gateway/restart-sentinel");
const OUTBOUND_RETRY_DELAY_MS = 1_000;
const OUTBOUND_MAX_ATTEMPTS = 45;
const RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS = process.env.VITEST ? 1 : 6_000;
const RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS = 20;
const CONTROL_PLANE_UPDATE_PENDING_RETRY_DELAY_MS = process.env.VITEST ? 1 : 2_000;
@@ -106,89 +105,19 @@ function enqueueRestartSentinelWake(
requestHeartbeat({ source: "restart-sentinel", intent: "immediate", reason: "wake", sessionKey });
}
async function waitForOutboundRetry(delayMs: number) {
async function waitForRetry(delayMs: number) {
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, delayMs);
timer.unref?.();
});
}
async function deliverRestartSentinelNotice(params: {
deps: CliDeps;
cfg: ReturnType<typeof loadSessionEntry>["cfg"];
sessionKey: string;
summary: string;
message: string;
channel: string;
to: string;
accountId?: string;
replyToId?: string;
threadId?: string;
session: ReturnType<typeof buildOutboundSessionContext>;
}) {
const payloads = [{ text: params.message }];
const queueId = await enqueueDelivery({
channel: params.channel,
to: params.to,
accountId: params.accountId,
replyToId: params.replyToId,
threadId: params.threadId,
payloads,
bestEffort: false,
}).catch(() => null);
for (let attempt = 1; attempt <= OUTBOUND_MAX_ATTEMPTS; attempt += 1) {
try {
const send = await sendDurableMessageBatch({
cfg: params.cfg,
channel: params.channel,
to: params.to,
accountId: params.accountId,
replyToId: params.replyToId,
threadId: params.threadId,
payloads,
session: params.session,
deps: params.deps,
bestEffort: false,
skipQueue: true,
});
if (send.status === "failed" || send.status === "partial_failed") {
throw send.error;
}
const results = send.status === "sent" ? send.results : [];
if (results.length > 0) {
if (queueId) {
await ackDelivery(queueId).catch(() => {});
}
return;
}
throw new Error("outbound delivery returned no results");
} catch (err) {
const retrying = attempt < OUTBOUND_MAX_ATTEMPTS;
const suffix = retrying ? `; retrying in ${OUTBOUND_RETRY_DELAY_MS}ms` : "";
log.warn(`${params.summary}: outbound delivery failed${suffix}: ${String(err)}`, {
channel: params.channel,
to: params.to,
sessionKey: params.sessionKey,
attempt,
maxAttempts: OUTBOUND_MAX_ATTEMPTS,
});
if (!retrying) {
if (queueId) {
await failDelivery(queueId, formatErrorMessage(err)).catch(() => undefined);
}
return;
}
await waitForOutboundRetry(OUTBOUND_RETRY_DELAY_MS);
}
}
}
function buildRestartContinuationMessageId(params: {
sessionKey: string;
kind: RestartSentinelContinuation["kind"];
ts: number;
revision: number;
}) {
return `restart-sentinel:${params.sessionKey}:${params.kind}:${params.ts}`;
return `restart-sentinel:${params.sessionKey}:${params.kind}:${params.revision}`;
}
function resolveRestartContinuationRoute(params: {
@@ -405,19 +334,22 @@ function buildQueuedRestartContinuation(params: {
continuation: RestartSentinelContinuation;
route?: SessionDeliveryRoute;
expectedSessionId?: string | undefined;
ts: number;
revision: number;
deliveryContext?: {
channel?: string;
to?: string;
accountId?: string;
threadId?: string | number;
};
idempotencyKey?: string;
}): QueuedSessionDeliveryPayload {
const idempotencyKey = buildRestartContinuationMessageId({
sessionKey: params.sessionKey,
kind: params.continuation.kind,
ts: params.ts,
});
const idempotencyKey =
params.idempotencyKey ??
buildRestartContinuationMessageId({
sessionKey: params.sessionKey,
kind: params.continuation.kind,
revision: params.revision,
});
if (params.continuation.kind === "systemEvent") {
return {
kind: "systemEvent",
@@ -426,6 +358,7 @@ function buildQueuedRestartContinuation(params: {
...(params.deliveryContext ? { deliveryContext: params.deliveryContext } : {}),
idempotencyKey,
maxRetries: RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS,
completionRetention: "permanent",
};
}
return {
@@ -435,6 +368,7 @@ function buildQueuedRestartContinuation(params: {
messageId: idempotencyKey,
...(params.expectedSessionId ? { expectedSessionId: params.expectedSessionId } : {}),
maxRetries: RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS,
completionRetention: "permanent",
...(params.route ? { route: params.route } : {}),
...(params.deliveryContext ? { deliveryContext: params.deliveryContext } : {}),
idempotencyKey,
@@ -474,7 +408,7 @@ async function drainRestartContinuationQueue(params: {
params.log.info(
`restart continuation: entry ${params.entryId} still waiting for the previous run to clear; retrying in ${RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS}ms`,
);
await waitForOutboundRetry(RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS);
await waitForRetry(RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS);
}
}
@@ -505,6 +439,7 @@ async function loadRestartSentinelStartupTask(params: {
return null;
}
const payload = sentinel.payload;
const sentinelRevision = sentinel.revision;
if (payload.kind === "update") {
recordLatestUpdateRestartSentinel(payload);
}
@@ -543,14 +478,25 @@ async function loadRestartSentinelStartupTask(params: {
if (!sessionKey) {
const mainSessionKey = resolveMainSessionKeyFromConfig();
enqueueSystemEvent(message, { sessionKey: mainSessionKey });
const wakeQueueId = await enqueueSessionDelivery(
buildQueuedRestartContinuation({
sessionKey: mainSessionKey,
continuation: { kind: "systemEvent", text: message },
revision: sentinelRevision,
idempotencyKey: `restart-sentinel-wake:${mainSessionKey}:${sentinelRevision}`,
}),
);
if (payload.continuation) {
log.warn(`${summary}: continuation skipped: restart sentinel sessionKey unavailable`, {
sessionKey: mainSessionKey,
continuationKind: payload.continuation.kind,
});
}
await clearRestartSentinel();
const consumed = await clearRestartSentinelIfRevision(sentinelRevision);
if (!consumed) {
log.info(`${summary}: newer restart sentinel preserved while draining durable wake`);
}
await drainRestartContinuationQueue({ deps: params.deps, entryId: wakeQueueId, log });
return { status: "ran" as const };
}
@@ -587,6 +533,9 @@ async function loadRestartSentinelStartupTask(params: {
let replyToId: string | undefined;
let resolvedThreadId = threadId;
let continuationQueueId: string | undefined;
let wakeQueueId: string | undefined;
let noticeQueueId: string | undefined;
let noticeQueueCreated = false;
let continuationRoute: SessionDeliveryRoute | undefined;
if (channel && to) {
@@ -624,11 +573,28 @@ async function loadRestartSentinelStartupTask(params: {
threadId: resolvedThreadId,
chatType,
});
}
const routedAgentTurnContinuation =
payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined;
if (!routedAgentTurnContinuation) {
wakeQueueId = await enqueueSessionDelivery(
buildQueuedRestartContinuation({
sessionKey: canonicalKey,
continuation: { kind: "systemEvent", text: message },
revision: sentinelRevision,
deliveryContext: wakeDeliveryContext,
idempotencyKey: `restart-sentinel-wake:${canonicalKey}:${sentinelRevision}`,
}),
);
}
if (payload.continuation) {
continuationQueueId = await enqueueSessionDelivery(
buildQueuedRestartContinuation({
sessionKey: canonicalKey,
continuation: payload.continuation,
ts: payload.ts,
revision: sentinelRevision,
route: continuationRoute,
expectedSessionId: entry?.sessionId,
deliveryContext:
@@ -644,19 +610,35 @@ async function loadRestartSentinelStartupTask(params: {
);
}
await clearRestartSentinel();
const routedAgentTurnContinuation =
payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined;
if (!routedAgentTurnContinuation) {
enqueueRestartSentinelWake(message, sessionKey, wakeDeliveryContext);
if (resolvedTo && channel) {
const queuedNotice = await enqueueRestartSentinelNotice({
channel,
to: resolvedTo,
accountId: origin?.accountId,
replyToId,
threadId: resolvedThreadId,
message,
sessionKey: canonicalKey,
revision: sentinelRevision,
});
noticeQueueId = queuedNotice.id;
noticeQueueCreated = queuedNotice.created;
}
if (resolvedTo && channel) {
const outboundSession = buildOutboundSessionContext({
cfg,
// Every downstream intent is durable before consuming the singleton. A
// failed or stale compare-delete cannot lose work or remove a newer row.
const consumed = await clearRestartSentinelIfRevision(sentinelRevision);
if (!consumed) {
log.info(`${summary}: newer restart sentinel preserved while draining durable work`, {
sessionKey: canonicalKey,
});
}
if (wakeQueueId) {
await drainRestartContinuationQueue({ deps: params.deps, entryId: wakeQueueId, log });
}
if (resolvedTo && channel && noticeQueueId && noticeQueueCreated) {
await deliverRestartSentinelNotice({
deps: params.deps,
cfg,
@@ -668,7 +650,11 @@ async function loadRestartSentinelStartupTask(params: {
accountId: origin?.accountId,
replyToId,
threadId: resolvedThreadId,
session: outboundSession,
queueId: noticeQueueId,
});
} else if (noticeQueueId && !noticeQueueCreated) {
log.info(`${summary}: durable restart notice already owned`, {
sessionKey: canonicalKey,
});
}
+39
View File
@@ -220,6 +220,45 @@ describe("delivery-queue-sqlite corrupt JSON resilience", () => {
last_error: null,
});
});
it("never prunes a permanent producer receipt", () => {
upsertDeliveryQueueEntry({
queueName: QUEUE,
entry: {
id: "rt-permanent",
enqueuedAt: 1,
retryCount: 0,
completionRetention: "permanent",
},
stateDir,
});
completeDeliveryQueueEntry(QUEUE, "rt-permanent", stateDir);
const { db } = openOpenClawStateDatabase({
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
db.prepare(
`UPDATE delivery_queue_entries
SET enqueued_at = ?
WHERE queue_name = ? AND id = ?`,
).run(Date.now() - 31 * 24 * 60 * 60_000, QUEUE, "rt-permanent");
enqueueValid("rt-prune-trigger");
completeDeliveryQueueEntry(QUEUE, "rt-prune-trigger", stateDir);
expect(getDeliveryQueueEntryStatus(QUEUE, "rt-permanent", stateDir)).toBe("completed");
const row = db
.prepare(
`SELECT recovery_state, entry_json
FROM delivery_queue_entries
WHERE queue_name = ? AND id = ?`,
)
.get(QUEUE, "rt-permanent") as Record<string, unknown>;
expect(row.recovery_state).toBe("completed_permanent");
expect(JSON.parse(String(row.entry_json))).toMatchObject({
completionRetention: "permanent",
recoveryState: "completed_permanent",
});
});
});
});
+76 -2
View File
@@ -13,6 +13,9 @@ import { runSqliteImmediateTransactionSync } from "./sqlite-transaction.js";
type QueueStatus = "pending" | "failed" | "completed";
type DeliveryQueueDatabase = Pick<OpenClawStateKyselyDatabase, "delivery_queue_entries">;
const COMPLETED_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
const PERMANENT_COMPLETION_RECOVERY_STATE = "completed_permanent";
export type DeliveryQueueCompletionRetention = "permanent";
/** Indexed metadata extracted from queue payloads for diagnostics and recovery. */
export type DeliveryQueueRowMetadata = {
@@ -28,6 +31,9 @@ export type DeliveryQueueEntryState = {
id: string;
enqueuedAt: number;
retryCount: number;
/** Durable delivery-call count reserved before invoking the provider path. */
attemptCount?: number;
completionRetention?: DeliveryQueueCompletionRetention;
acknowledgedAt?: number;
lastAttemptAt?: number;
lastError?: string;
@@ -433,11 +439,19 @@ export function deleteDeliveryQueueEntry(queueName: string, id: string, stateDir
/** Retain a delivered row as a durable idempotency tombstone. */
export function completeDeliveryQueueEntry(queueName: string, id: string, stateDir?: string): void {
const now = Date.now();
const current = loadDeliveryQueueEntry(queueName, id, stateDir);
const retainPermanently = current?.completionRetention === "permanent";
const tombstone = {
id,
enqueuedAt: now,
retryCount: 0,
acknowledgedAt: now,
...(retainPermanently
? {
completionRetention: "permanent" as const,
recoveryState: PERMANENT_COMPLETION_RECOVERY_STATE,
}
: {}),
};
const completed = upsertDeliveryQueueEntry({
queueName,
@@ -453,7 +467,8 @@ export function completeDeliveryQueueEntry(queueName: string, id: string, stateD
}
throw enoent(queueName, id);
}
// Thirty days covers delayed producer replays while bounding successful-row growth.
// Ordinary receipts expire after thirty days. Permanent producer receipts
// survive because their source intent can outlive any bounded retry window.
const database = openStateDatabase(stateDir);
const queueDb = getNodeSqliteKysely<DeliveryQueueDatabase>(database.db);
executeSqliteQuerySync(
@@ -462,7 +477,13 @@ export function completeDeliveryQueueEntry(queueName: string, id: string, stateD
.deleteFrom("delivery_queue_entries")
.where("queue_name", "=", queueName)
.where("status", "=", "completed")
.where("enqueued_at", "<", now - COMPLETED_TOMBSTONE_RETENTION_MS),
.where("enqueued_at", "<", now - COMPLETED_TOMBSTONE_RETENTION_MS)
.where((eb) =>
eb.or([
eb("recovery_state", "is", null),
eb("recovery_state", "!=", PERMANENT_COMPLETION_RECOVERY_STATE),
]),
),
);
}
@@ -480,6 +501,59 @@ export function updateDeliveryQueueEntry(
upsertDeliveryQueueEntry({ queueName, entry: update(current), stateDir });
}
type ReserveDeliveryQueueAttemptResult =
| { status: "reserved"; attemptCount: number }
| { status: "exhausted"; attemptCount: number };
/** Atomically reserve one provider-delivery call before executing it. */
export function reserveDeliveryQueueEntryAttempt(params: {
queueName: string;
id: string;
maxAttempts: number;
stateDir?: string;
}): ReserveDeliveryQueueAttemptResult {
if (!Number.isInteger(params.maxAttempts) || params.maxAttempts <= 0) {
throw new Error(`Invalid delivery attempt budget: ${params.maxAttempts}`);
}
const database = openStateDatabase(params.stateDir);
return runSqliteImmediateTransactionSync(
database.db,
() => {
const current = loadDeliveryQueueEntry(params.queueName, params.id, params.stateDir);
if (!current) {
throw enoent(params.queueName, params.id);
}
const persistedAttemptCount =
typeof current.attemptCount === "number" &&
Number.isInteger(current.attemptCount) &&
current.attemptCount >= 0
? current.attemptCount
: 0;
const attemptCount = Math.max(persistedAttemptCount, current.retryCount);
if (attemptCount >= params.maxAttempts) {
return { status: "exhausted", attemptCount };
}
const reservedAttemptCount = attemptCount + 1;
const updated = upsertDeliveryQueueEntryInDatabase(
{
queueName: params.queueName,
entry: { ...current, attemptCount: reservedAttemptCount },
updatePendingOnly: true,
},
database,
);
if (!updated) {
throw enoent(params.queueName, params.id);
}
return { status: "reserved", attemptCount: reservedAttemptCount };
},
{
databaseLabel: "openclaw-state",
operationLabel: `reserve ${params.queueName} delivery attempt`,
},
);
}
/** Dead-lettered entry counts for one queue namespace. */
type FailedDeliveryQueueCount = {
queueName: string;
+57 -13
View File
@@ -52,6 +52,7 @@ import {
loadPendingDeliveries,
markDeliveryPlatformOutcomeUnknown,
moveToFailed,
reserveDeliveryAttempt,
type QueuedDelivery,
type QueuedDeliveryPayload,
} from "./delivery-queue-storage.js";
@@ -98,7 +99,7 @@ type ActiveDeliveryClaimResult<T> =
| { status: "claimed"; value: T }
| { status: "claimed-by-other-owner" };
const MAX_RETRIES = 5;
const DEFAULT_MAX_RETRIES = 5;
const PERMANENT_ERROR_PATTERNS: readonly RegExp[] = [
/no conversation reference found/i,
@@ -118,6 +119,20 @@ const drainInProgress = new Map<string, boolean>();
const entriesInProgress = new Set<string>();
const recoveryReplayPacer = createRecoveryReplayPacer();
function resolveMaxRetries(entry: QueuedDelivery): number {
const configured = entry.maxRetries;
return typeof configured === "number" && Number.isInteger(configured) && configured > 0
? configured
: DEFAULT_MAX_RETRIES;
}
function resolveAttemptCount(entry: QueuedDelivery): number {
const persisted = entry.attemptCount;
const attemptCount =
typeof persisted === "number" && Number.isInteger(persisted) && persisted >= 0 ? persisted : 0;
return Math.max(attemptCount, entry.retryCount);
}
function resolveRecoveryDeadlineMs(maxRecoveryMs: number | undefined): number {
const durationMs =
typeof maxRecoveryMs === "number" && Number.isFinite(maxRecoveryMs)
@@ -150,10 +165,14 @@ function emitQueuedAuditTerminals(
});
}
function needsUnknownSendReconciliation(entry: QueuedDelivery): boolean {
return (
entry.recoveryState === "send_attempt_started" || entry.recoveryState === "unknown_after_send"
);
}
function queuedDeadLetterAuditTerminals(entry: QueuedDelivery) {
const ambiguous =
entry.recoveryState === "send_attempt_started" || entry.recoveryState === "unknown_after_send";
if (ambiguous) {
if (needsUnknownSendReconciliation(entry)) {
return uniformOutboundAuditTerminals(entry.payloads.length, {
outcome: "unknown",
failureStage: "queue",
@@ -597,14 +616,13 @@ async function drainQueuedEntry(opts: {
onFailed?: (entry: QueuedDelivery, errMsg: string) => void;
}): Promise<"recovered" | "failed" | "moved-to-failed" | "already-gone"> {
const { entry } = opts;
const maxRetries = resolveMaxRetries(entry);
const attemptBudgetExhausted = resolveAttemptCount(entry) >= maxRetries;
const ownerState = await resolveCompletedOwnerBeforeRecovery(opts);
if (ownerState !== "continue") {
return ownerState;
}
if (
entry.recoveryState === "send_attempt_started" ||
entry.recoveryState === "unknown_after_send"
) {
if (needsUnknownSendReconciliation(entry)) {
// A crash after platform send start cannot be blindly replayed; adapters
// must reconcile whether the platform already committed the message.
const reconciliation = await reconcileUnknownQueuedDelivery({
@@ -668,7 +686,11 @@ async function drainQueuedEntry(opts: {
}
opts.log.warn(`Delivery entry ${entry.id} ${errMsg}`);
opts.onFailed?.(entry, errMsg);
if (reconciliation?.status === "unresolved" && reconciliation.retryable === true) {
if (
reconciliation?.status === "unresolved" &&
reconciliation.retryable === true &&
!attemptBudgetExhausted
) {
try {
await failDelivery(entry.id, errMsg, opts.stateDir);
return "failed";
@@ -715,6 +737,22 @@ async function drainQueuedEntry(opts: {
commitHooksRun = true;
await runOutboundDeliveryCommitHooks(deliveredResults);
};
const reservation = await reserveDeliveryAttempt(entry.id, maxRetries, opts.stateDir);
if (reservation.status === "exhausted") {
const errMsg = `delivery retry budget exhausted (${reservation.attemptCount}/${maxRetries})`;
markDurableDeliveryFailedBestEffort(entry, opts.log);
try {
await moveToFailed(entry.id, opts.stateDir);
} catch (moveErr) {
if (getErrnoCode(moveErr) === "ENOENT") {
return "already-gone";
}
throw moveErr;
}
emitQueuedAuditTerminals(entry, () => queuedDeadLetterAuditTerminals(entry));
opts.onFailed?.(entry, errMsg);
return "moved-to-failed";
}
const recoverySpoolPaths = collectEntrySpoolPaths(entry.payloads, opts.stateDir);
let mediaRecoveryLeaseId: string | undefined;
try {
@@ -980,7 +1018,11 @@ export async function drainPendingDeliveries(opts: {
continue;
}
if (currentEntry.retryCount >= MAX_RETRIES) {
const maxRetries = resolveMaxRetries(currentEntry);
if (
resolveAttemptCount(currentEntry) >= maxRetries &&
!needsUnknownSendReconciliation(currentEntry)
) {
try {
markDurableDeliveryFailedBestEffort(currentEntry, opts.log);
await moveToFailed(currentEntry.id, opts.stateDir);
@@ -1044,7 +1086,7 @@ export async function drainPendingDeliveries(opts: {
/**
* On gateway startup, scan the delivery queue and retry any pending entries.
* Uses exponential backoff and moves entries that exceed MAX_RETRIES to failed/.
* Uses exponential backoff and moves entries that exhaust their retry budget to failed/.
*/
export async function recoverPendingDeliveries(opts: {
deliver: DeliverFn;
@@ -1100,9 +1142,11 @@ export async function recoverPendingDeliveries(opts: {
continue;
}
if (currentEntry.retryCount >= MAX_RETRIES) {
const maxRetries = resolveMaxRetries(currentEntry);
const attemptCount = resolveAttemptCount(currentEntry);
if (attemptCount >= maxRetries && !needsUnknownSendReconciliation(currentEntry)) {
opts.log.warn(
`Delivery ${currentEntry.id} exceeded max retries (${currentEntry.retryCount}/${MAX_RETRIES}) — moving to failed/`,
`Delivery ${currentEntry.id} exceeded max retries (${attemptCount}/${maxRetries}) — moving to failed/`,
);
const movedToFailed = await moveEntryToFailedWithLogging(
currentEntry,
+33 -3
View File
@@ -6,6 +6,7 @@ import type { RenderedMessageBatchPlanItem } from "../../channels/message/types.
import type { ReplyToMode } from "../../config/types.js";
import type { PluginHookReplyPayloadSendingContext } from "../../plugins/hook-types.js";
import {
completeDeliveryQueueEntry,
commitStagedDeliveryQueueEntry,
commitStagedDeliveryQueueEntryOnce,
deleteDeliveryQueueEntry,
@@ -13,9 +14,11 @@ import {
loadDeliveryQueueEntries,
loadDeliveryQueueEntry,
moveDeliveryQueueEntryToFailed,
reserveDeliveryQueueEntryAttempt,
updateDeliveryQueueEntry,
upsertDeliveryQueueEntry,
type DeliveryQueueRowMetadata,
type DeliveryQueueCompletionRetention,
} from "../delivery-queue-sqlite.js";
import { generateSecureUuid } from "../secure-random.js";
import type { DurableDeliveryCompletion } from "./delivery-completion.js";
@@ -85,12 +88,17 @@ export type QueuedDeliveryPayload = {
preparedMessageId?: string;
/** Serializable owner state finalized by both live delivery and recovery. */
deliveryCompletion?: DurableDeliveryCompletion;
/** Retain a terminal receipt when the producer may replay this stable intent indefinitely. */
completionRetention?: DeliveryQueueCompletionRetention;
/** Producer-specific retry budget; omitted entries use the queue default. */
maxRetries?: number;
};
export interface QueuedDelivery extends QueuedDeliveryPayload {
id: string;
enqueuedAt: number;
retryCount: number;
attemptCount: number;
lastAttemptAt?: number;
lastError?: string;
platformSendStartedAt?: number;
@@ -135,7 +143,10 @@ function createQueuedDelivery(params: QueuedDeliveryPayload, id: string): Queued
gatewayClientScopes: params.gatewayClientScopes,
preparedMessageId: params.preparedMessageId,
deliveryCompletion: params.deliveryCompletion,
completionRetention: params.completionRetention,
maxRetries: params.maxRetries,
retryCount: 0,
attemptCount: 0,
};
}
@@ -224,7 +235,7 @@ type AckDeliveryOptions = {
retainSpoolArtifacts?: boolean;
};
/** Remove a successfully delivered entry from the queue. */
/** Remove a successfully delivered entry, or retain its permanent producer receipt. */
export async function ackDelivery(
id: string,
stateDir?: string,
@@ -233,8 +244,17 @@ export async function ackDelivery(
// Read the media references before the row goes, then unlink only after the
// delete commits. A crash in between leaves an orphan for the retention sweep;
// unlinking first could strip media from a row that still has to replay.
const spoolPaths = loadEntrySpoolPaths(id, stateDir);
deleteDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir);
const entry = loadDeliveryQueueEntry(
OUTBOUND_DELIVERY_QUEUE_NAME,
id,
stateDir,
) as QueuedDelivery | null;
const spoolPaths = entry ? collectEntrySpoolPaths(entry.payloads, stateDir) : [];
if (entry?.completionRetention === "permanent") {
completeDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir);
} else {
deleteDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir);
}
if (!options?.retainSpoolArtifacts) {
await releaseSpoolArtifacts(spoolPaths, stateDir);
}
@@ -283,6 +303,16 @@ export async function failDeliveryAfterPlatformSend(
}));
}
/** Reserve one durable delivery call before invoking the provider path. */
export async function reserveDeliveryAttempt(id: string, maxAttempts: number, stateDir?: string) {
return reserveDeliveryQueueEntryAttempt({
queueName: OUTBOUND_DELIVERY_QUEUE_NAME,
id,
maxAttempts,
stateDir,
});
}
function updateQueuedDelivery(
id: string,
stateDir: string | undefined,
@@ -4,13 +4,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { controlNextRecoverySleep } from "../../../test/helpers/infra/delivery-recovery.js";
import type { OpenClawConfig } from "../../config/config.js";
import { openOpenClawStateDatabase } from "../../state/openclaw-state-db.js";
import { loadPendingDeliveries } from "./delivery-queue-storage.js";
import { loadPendingDeliveries, reserveDeliveryAttempt } from "./delivery-queue-storage.js";
import {
type DeliverFn,
drainPendingDeliveries,
enqueueDelivery,
failDelivery,
markDeliveryPlatformOutcomeUnknown,
markDeliveryPlatformSendAttemptStarted,
type RecoveryLogger,
recoverPendingDeliveries,
withActiveDeliveryClaim,
@@ -27,8 +28,12 @@ const MAX_RETRIES = 5;
const stubCfg = {} as OpenClawConfig;
const NO_LISTENER_ERROR = "No active DirectChat listener";
const sleepMock = vi.hoisted(() => vi.fn<(ms: number) => Promise<void>>());
const resolveOutboundChannelMessageAdapterMock = vi.hoisted(() => vi.fn());
vi.mock("../../utils/sleep.js", () => ({ sleep: sleepMock }));
vi.mock("./channel-resolution.js", () => ({
resolveOutboundChannelMessageAdapter: resolveOutboundChannelMessageAdapterMock,
}));
function normalizeReconnectAccountIdForTest(accountId?: string | null): string {
return (accountId ?? "").trim() || "default";
@@ -146,6 +151,7 @@ describe("drainPendingDeliveries for reconnect", () => {
tmpDir = fixtures.tmpDir();
sleepMock.mockReset();
sleepMock.mockResolvedValue(undefined);
resolveOutboundChannelMessageAdapterMock.mockReset();
});
it("drains entries that failed with 'no listener' error", async () => {
@@ -218,6 +224,46 @@ describe("drainPendingDeliveries for reconnect", () => {
expectLogMessageWith(log.warn, "refusing blind replay without adapter reconciliation");
});
it("reconciles an exhausted final attempt before dead-lettering", async () => {
const log = createRecoveryLog();
const deliver = vi.fn<DeliverFn>(async () => {});
const id = await enqueueDelivery(
{
channel: "directchat",
to: "+1555",
payloads: [{ text: "maybe sent" }],
accountId: "acct1",
maxRetries: 1,
},
tmpDir,
);
await reserveDeliveryAttempt(id, 1, tmpDir);
await markDeliveryPlatformSendAttemptStarted(id, tmpDir);
const reconcileUnknownSend = vi.fn().mockResolvedValue({
status: "sent",
messageId: "platform-final",
receipt: {
primaryPlatformMessageId: "platform-final",
platformMessageIds: ["platform-final"],
parts: [{ platformMessageId: "platform-final", kind: "text", index: 0 }],
sentAt: 1,
},
});
resolveOutboundChannelMessageAdapterMock.mockReturnValue({
durableFinal: {
capabilities: { reconcileUnknownSend: true },
reconcileUnknownSend,
},
});
await drainAcct1DirectChatReconnect({ deliver, log, stateDir: tmpDir });
expect(deliver).not.toHaveBeenCalled();
expect(reconcileUnknownSend).toHaveBeenCalledOnce();
expect(await loadPendingDeliveries(tmpDir)).toHaveLength(0);
expect(readOutboundQueueStatus(tmpDir, id)).toBeUndefined();
});
it("skips entries where retryCount >= MAX_RETRIES", async () => {
const log = createRecoveryLog();
const deliver = vi.fn<DeliverFn>(async () => {});
@@ -24,7 +24,7 @@ import {
} from "./deliver-types.js";
import { attachOutboundDeliveryCommitHook } from "./delivery-commit-hooks.js";
import { pruneOrphanedDeliveryQueueMedia } from "./delivery-queue-media-spool.js";
import { loadPendingDeliveries } from "./delivery-queue-storage.js";
import { loadPendingDeliveries, reserveDeliveryAttempt } from "./delivery-queue-storage.js";
import {
ackDelivery,
enqueueDelivery,
@@ -498,6 +498,67 @@ describe("delivery-queue recovery", () => {
});
});
it("honors a producer-specific retry budget", async () => {
const id = await enqueueDelivery(
{
channel: "demo-channel-a",
to: "+1",
payloads: [{ text: "a" }],
maxRetries: 45,
},
tmpDir(),
);
setQueuedEntryState(tmpDir(), id, {
retryCount: MAX_RETRIES,
lastAttemptAt: Date.now() - 10_000_000,
});
const deliver = vi.fn().mockResolvedValue([]);
const { result } = await runRecovery({ deliver });
expect(deliver).toHaveBeenCalledOnce();
expect(result).toMatchObject({ recovered: 1, skippedMaxRetries: 0 });
expect(readOutboundQueueStatus(tmpDir(), id)).toBeUndefined();
});
it("dead-letters an atomically exhausted attempt budget before replay", async () => {
const id = await enqueueDelivery(
{
channel: "demo-channel-a",
to: "+1",
payloads: [{ text: "a" }],
maxRetries: 1,
},
tmpDir(),
);
await reserveDeliveryAttempt(id, 1, tmpDir());
const deliver = vi.fn();
const { result } = await runRecovery({ deliver });
expect(deliver).not.toHaveBeenCalled();
expect(result.skippedMaxRetries).toBe(1);
expect(readOutboundQueueStatus(tmpDir(), id)).toBe("failed");
});
it("ignores an invalid producer retry budget", async () => {
await enqueueDelivery(
{
channel: "demo-channel-a",
to: "+1",
payloads: [{ text: "a" }],
maxRetries: 0.5,
},
tmpDir(),
);
const deliver = vi.fn().mockResolvedValue([]);
const { result } = await runRecovery({ deliver });
expect(deliver).toHaveBeenCalledOnce();
expect(result).toMatchObject({ recovered: 1, skippedMaxRetries: 0 });
});
it("dead-letters max-retry entries even when conversation owner state is missing", async () => {
const storePath = path.join(tmpDir(), "missing-owner-sessions.json");
const id = await enqueueDelivery(
@@ -537,6 +598,7 @@ describe("delivery-queue recovery", () => {
);
setQueuedEntryState(tmpDir(), id, {
retryCount: MAX_RETRIES,
lastAttemptAt: Date.now() - 10_000_000,
platformSendStartedAt: Date.now(),
recoveryState: "send_attempt_started",
});
@@ -544,7 +606,7 @@ describe("delivery-queue recovery", () => {
const { result } = await runRecovery({ deliver: vi.fn() });
unsubscribe();
expect(result.skippedMaxRetries).toBe(1);
expect(result).toMatchObject({ failed: 1, skippedMaxRetries: 0 });
expect(auditEvents).toHaveLength(1);
expect(auditEvents[0]).toMatchObject({
sourceId: `message:outbound:queue:${id}:payload:0`,
@@ -571,6 +633,7 @@ describe("delivery-queue recovery", () => {
const entries = await loadPendingDeliveries(tmpDir());
expect(entries).toHaveLength(1);
expect(entries[0]?.retryCount).toBe(1);
expect(entries[0]?.attemptCount).toBe(1);
expect(entries[0]?.lastError).toBe("network down");
expect(auditEvents).toEqual([]);
});
@@ -1008,9 +1071,11 @@ describe("delivery-queue recovery", () => {
replyToId: "root-message",
threadId: "thread-1",
silent: true,
maxRetries: 1,
},
tmpDir(),
);
await reserveDeliveryAttempt(id, 1, tmpDir());
await markDeliveryPlatformSendAttemptStarted(id, tmpDir(), {
replyToId: "hooked-root-message",
});
@@ -1172,6 +1237,45 @@ describe("delivery-queue recovery", () => {
expect(entries[0]?.lastError).toContain("provider lookup timed out");
});
it("dead-letters an exhausted unknown send after retryable reconciliation fails", async () => {
const id = await enqueueDelivery(
{
channel: "demo-channel-a",
to: "+1",
payloads: [{ text: "unknown final attempt" }],
maxRetries: 1,
},
tmpDir(),
);
await reserveDeliveryAttempt(id, 1, tmpDir());
setQueuedEntryState(tmpDir(), id, {
retryCount: 0,
lastAttemptAt: Date.now() - 10_000_000,
platformSendStartedAt: Date.now(),
recoveryState: "unknown_after_send",
});
const reconcileUnknownSend = vi.fn().mockResolvedValue({
status: "unresolved",
error: "provider lookup timed out",
retryable: true,
});
resolveOutboundChannelMessageAdapterMock.mockReturnValue({
durableFinal: {
capabilities: { reconcileUnknownSend: true },
reconcileUnknownSend,
},
});
const deliver = vi.fn().mockResolvedValue([]);
const { result } = await runRecovery({ deliver });
expect(reconcileUnknownSend).toHaveBeenCalledOnce();
expect(deliver).not.toHaveBeenCalled();
expect(result).toMatchObject({ failed: 1, skippedMaxRetries: 0 });
expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0);
expect(readOutboundQueueStatus(tmpDir(), id)).toBe("failed");
});
it("does not reconcile unknown-after-send entries unless the adapter declares the capability", async () => {
const id = await enqueueDelivery(
{ channel: "demo-channel-a", to: "+1", payloads: [{ text: "hidden method" }] },
@@ -9,6 +9,7 @@ import {
loadPendingDelivery,
loadPendingDeliveries,
moveToFailed,
reserveDeliveryAttempt,
} from "./delivery-queue-storage.js";
import {
ackDelivery,
@@ -55,6 +56,40 @@ describe("delivery-queue storage", () => {
}
describe("enqueue + ack lifecycle", () => {
it("persists a producer-specific retry budget", async () => {
const id = await enqueueTextDelivery({
channel: "directchat",
to: "+1555",
payloads: [{ text: "retry-budget" }],
maxRetries: 45,
});
expect(readQueuedEntry(tmpDir(), id).maxRetries).toBe(45);
});
it("atomically reserves delivery attempts up to the producer budget", async () => {
const id = await enqueueTextDelivery({
channel: "directchat",
to: "+1555",
payloads: [{ text: "attempt-budget" }],
maxRetries: 2,
});
await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({
status: "reserved",
attemptCount: 1,
});
await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({
status: "reserved",
attemptCount: 2,
});
await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({
status: "exhausted",
attemptCount: 2,
});
expect(readQueuedEntry(tmpDir(), id).attemptCount).toBe(2);
});
it("creates and removes a queue entry", async () => {
const id = await enqueueTextDelivery(
{
@@ -180,6 +215,36 @@ describe("delivery-queue storage", () => {
});
});
it("keeps permanent completion ownership after ack", async () => {
const id = "restart-sentinel-notice:agent:main:main:123";
await enqueueDeliveryOnce(
{
channel: "directchat",
to: "+1555",
payloads: [{ text: "restart complete" }],
completionRetention: "permanent",
},
id,
tmpDir(),
);
await ackDelivery(id, tmpDir());
const repeated = await enqueueDeliveryOnce(
{
channel: "directchat",
to: "+1555",
payloads: [{ text: "must not replay" }],
completionRetention: "permanent",
},
id,
tmpDir(),
);
expect(repeated).toEqual({ id, created: false });
expect(await loadPendingDeliveries(tmpDir())).toEqual([]);
expect(readStatus(id)).toBe("completed");
});
it("ack is idempotent (no error on missing file)", async () => {
await expect(ackDelivery("nonexistent-id", tmpDir())).resolves.toBeUndefined();
});
+660
View File
@@ -0,0 +1,660 @@
import type { DatabaseSync } from "node:sqlite";
import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
type RestartSentinelLog = {
stdoutTail?: string | null;
stderrTail?: string | null;
exitCode?: number | null;
};
type RestartSentinelStep = {
name: string;
command: string;
cwd?: string | null;
durationMs?: number | null;
log?: RestartSentinelLog | null;
};
type RestartSentinelStats = {
mode?: string;
root?: string;
requiresRestart?: boolean;
handoffId?: string;
before?: Record<string, unknown> | null;
after?: Record<string, unknown> | null;
steps?: RestartSentinelStep[];
reason?: string | null;
durationMs?: number | null;
};
export type RestartSentinelContinuation =
| {
kind: "systemEvent";
text: string;
}
| {
kind: "agentTurn";
message: string;
};
export type RestartSentinelPayload = {
kind: "config-apply" | "config-auto-recovery" | "config-patch" | "update" | "restart";
status: "ok" | "error" | "skipped";
ts: number;
sessionKey?: string;
deliveryContext?: {
channel?: string;
to?: string;
accountId?: string;
};
threadId?: string;
message?: string | null;
continuation?: RestartSentinelContinuation | null;
doctorHint?: string | null;
stats?: RestartSentinelStats | null;
};
export type RestartSentinelEnvelope = {
version: 1;
payload: RestartSentinelPayload;
};
export type RestartSentinel = RestartSentinelEnvelope & {
/** Optimistic-concurrency revision backed by gateway_restart_sentinel.updated_at_ms. */
revision: number;
};
type RestartSentinelRowState =
| { kind: "missing" }
| { kind: "invalid"; revision: number }
| { kind: "valid"; sentinel: RestartSentinel };
const RESTART_SENTINEL_KEY = "current";
const RESTART_SENTINEL_REVISION_FLOOR_KEY = "revision-floor";
const RESTART_SENTINEL_KINDS = new Set<RestartSentinelPayload["kind"]>([
"config-apply",
"config-auto-recovery",
"config-patch",
"update",
"restart",
]);
const RESTART_SENTINEL_STATUSES = new Set<RestartSentinelPayload["status"]>([
"ok",
"error",
"skipped",
]);
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
function isFiniteNumber(value: unknown): value is number {
return typeof value === "number" && Number.isFinite(value);
}
function isSafeInteger(value: unknown): value is number {
return typeof value === "number" && Number.isSafeInteger(value);
}
function parseOptionalNullableString(
record: Record<string, unknown>,
key: string,
): string | null | undefined | false {
const value = record[key];
if (value === undefined || value === null || typeof value === "string") {
return value;
}
return false;
}
function parseRestartSentinelLog(value: unknown): RestartSentinelLog | null {
if (!isPlainRecord(value)) {
return null;
}
const stdoutTail = parseOptionalNullableString(value, "stdoutTail");
const stderrTail = parseOptionalNullableString(value, "stderrTail");
const exitCode = value.exitCode;
if (
stdoutTail === false ||
stderrTail === false ||
(exitCode !== undefined && exitCode !== null && !isSafeInteger(exitCode))
) {
return null;
}
const result: RestartSentinelLog = {};
if (stdoutTail !== undefined) {
result.stdoutTail = stdoutTail;
}
if (stderrTail !== undefined) {
result.stderrTail = stderrTail;
}
if (exitCode !== undefined) {
result.exitCode = exitCode as number | null;
}
return result;
}
function parseRestartSentinelStep(value: unknown): RestartSentinelStep | null {
if (
!isPlainRecord(value) ||
typeof value.name !== "string" ||
typeof value.command !== "string"
) {
return null;
}
const cwd = parseOptionalNullableString(value, "cwd");
const durationMs = value.durationMs;
const log = value.log;
if (
cwd === false ||
(durationMs !== undefined && durationMs !== null && !isFiniteNumber(durationMs)) ||
(log !== undefined && log !== null && !parseRestartSentinelLog(log))
) {
return null;
}
const result: RestartSentinelStep = { name: value.name, command: value.command };
if (cwd !== undefined) {
result.cwd = cwd;
}
if (durationMs !== undefined) {
result.durationMs = durationMs as number | null;
}
if (log !== undefined) {
result.log = log === null ? null : parseRestartSentinelLog(log);
}
return result;
}
function parseRestartSentinelStats(value: unknown): RestartSentinelStats | null {
if (!isPlainRecord(value)) {
return null;
}
const mode = parseOptionalNullableString(value, "mode");
const root = parseOptionalNullableString(value, "root");
const handoffId = parseOptionalNullableString(value, "handoffId");
const reason = parseOptionalNullableString(value, "reason");
const before = value.before;
const after = value.after;
const steps = value.steps;
const durationMs = value.durationMs;
if (
mode === false ||
mode === null ||
root === false ||
root === null ||
handoffId === false ||
handoffId === null ||
reason === false ||
(value.requiresRestart !== undefined && typeof value.requiresRestart !== "boolean") ||
(before !== undefined && before !== null && !isPlainRecord(before)) ||
(after !== undefined && after !== null && !isPlainRecord(after)) ||
(steps !== undefined &&
(!Array.isArray(steps) || steps.some((step) => !parseRestartSentinelStep(step)))) ||
(durationMs !== undefined && durationMs !== null && !isFiniteNumber(durationMs))
) {
return null;
}
const result: RestartSentinelStats = {};
if (mode !== undefined) {
result.mode = mode;
}
if (root !== undefined) {
result.root = root;
}
if (value.requiresRestart !== undefined) {
result.requiresRestart = value.requiresRestart as boolean;
}
if (handoffId !== undefined) {
result.handoffId = handoffId;
}
if (before !== undefined) {
result.before = before as Record<string, unknown> | null;
}
if (after !== undefined) {
result.after = after as Record<string, unknown> | null;
}
if (steps !== undefined) {
result.steps = steps.map((step) => parseRestartSentinelStep(step)!);
}
if (reason !== undefined) {
result.reason = reason;
}
if (durationMs !== undefined) {
result.durationMs = durationMs as number | null;
}
return result;
}
function parseRestartSentinelContinuation(value: unknown): RestartSentinelContinuation | null {
if (!isPlainRecord(value)) {
return null;
}
if (value.kind === "systemEvent" && typeof value.text === "string") {
return { kind: "systemEvent", text: value.text };
}
if (value.kind === "agentTurn" && typeof value.message === "string") {
return { kind: "agentTurn", message: value.message };
}
return null;
}
function parseRestartSentinelPayload(value: unknown): RestartSentinelPayload | null {
if (
!isPlainRecord(value) ||
!RESTART_SENTINEL_KINDS.has(value.kind as RestartSentinelPayload["kind"]) ||
!RESTART_SENTINEL_STATUSES.has(value.status as RestartSentinelPayload["status"]) ||
!isSafeInteger(value.ts)
) {
return null;
}
const sessionKey = parseOptionalNullableString(value, "sessionKey");
const threadId = parseOptionalNullableString(value, "threadId");
const message = parseOptionalNullableString(value, "message");
const doctorHint = parseOptionalNullableString(value, "doctorHint");
if (
sessionKey === false ||
sessionKey === null ||
threadId === false ||
threadId === null ||
message === false ||
doctorHint === false
) {
return null;
}
let deliveryContext: RestartSentinelPayload["deliveryContext"];
if (value.deliveryContext !== undefined) {
if (!isPlainRecord(value.deliveryContext)) {
return null;
}
const channel = parseOptionalNullableString(value.deliveryContext, "channel");
const to = parseOptionalNullableString(value.deliveryContext, "to");
const accountId = parseOptionalNullableString(value.deliveryContext, "accountId");
if (
channel === false ||
channel === null ||
to === false ||
to === null ||
accountId === false ||
accountId === null
) {
return null;
}
deliveryContext = {};
if (channel !== undefined) {
deliveryContext.channel = channel;
}
if (to !== undefined) {
deliveryContext.to = to;
}
if (accountId !== undefined) {
deliveryContext.accountId = accountId;
}
}
let continuation: RestartSentinelContinuation | null | undefined;
if (value.continuation !== undefined) {
continuation =
value.continuation === null ? null : parseRestartSentinelContinuation(value.continuation);
if (continuation === null && value.continuation !== null) {
return null;
}
}
let stats: RestartSentinelStats | null | undefined;
if (value.stats !== undefined) {
stats = value.stats === null ? null : parseRestartSentinelStats(value.stats);
if (stats === null && value.stats !== null) {
return null;
}
}
const result: RestartSentinelPayload = {
kind: value.kind as RestartSentinelPayload["kind"],
status: value.status as RestartSentinelPayload["status"],
ts: value.ts,
};
if (sessionKey !== undefined) {
result.sessionKey = sessionKey;
}
// SQL NULL is canonical absence for optional top-level columns. Normalize
// legacy nulls and empty routes so writes and typed-column reads agree.
if (deliveryContext !== undefined && Object.keys(deliveryContext).length > 0) {
result.deliveryContext = deliveryContext;
}
if (threadId !== undefined) {
result.threadId = threadId;
}
if (message !== undefined && message !== null) {
result.message = message;
}
if (continuation !== undefined && continuation !== null) {
result.continuation = continuation;
}
if (doctorHint !== undefined && doctorHint !== null) {
result.doctorHint = doctorHint;
}
if (stats !== undefined && stats !== null) {
result.stats = stats;
}
return result;
}
export function parseRestartSentinelEnvelope(value: unknown): RestartSentinelEnvelope | null {
if (!isPlainRecord(value) || value.version !== 1) {
return null;
}
const payload = parseRestartSentinelPayload(value.payload);
return payload ? { version: 1, payload } : null;
}
function parseRequiredJson(value: string | null): unknown {
if (value === null) {
return undefined;
}
try {
return JSON.parse(value) as unknown;
} catch {
return undefined;
}
}
function decodeRestartSentinelRow(row: {
version: number;
kind: string;
status: string;
ts: number;
session_key: string | null;
thread_id: string | null;
delivery_channel: string | null;
delivery_to: string | null;
delivery_account_id: string | null;
message: string | null;
continuation_json: string | null;
doctor_hint: string | null;
stats_json: string | null;
updated_at_ms: number;
}): RestartSentinel | null {
if (row.version !== 1 || !isSafeInteger(row.updated_at_ms)) {
return null;
}
const candidate: Record<string, unknown> = {
kind: row.kind,
status: row.status,
ts: row.ts,
};
if (row.session_key !== null) {
candidate.sessionKey = row.session_key;
}
if (row.thread_id !== null) {
candidate.threadId = row.thread_id;
}
if (
row.delivery_channel !== null ||
row.delivery_to !== null ||
row.delivery_account_id !== null
) {
candidate.deliveryContext = {
...(row.delivery_channel === null ? {} : { channel: row.delivery_channel }),
...(row.delivery_to === null ? {} : { to: row.delivery_to }),
...(row.delivery_account_id === null ? {} : { accountId: row.delivery_account_id }),
};
}
if (row.message !== null) {
candidate.message = row.message;
}
if (row.continuation_json !== null) {
const continuation = parseRequiredJson(row.continuation_json);
if (continuation === undefined) {
return null;
}
candidate.continuation = continuation;
}
if (row.doctor_hint !== null) {
candidate.doctorHint = row.doctor_hint;
}
if (row.stats_json !== null) {
const stats = parseRequiredJson(row.stats_json);
if (stats === undefined) {
return null;
}
candidate.stats = stats;
}
const payload = parseRestartSentinelPayload(candidate);
return payload ? { version: 1, payload, revision: row.updated_at_ms } : null;
}
export function readRestartSentinelRowSync(db: DatabaseSync): RestartSentinelRowState {
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
const row = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select([
"version",
"kind",
"status",
"ts",
"session_key",
"thread_id",
"delivery_channel",
"delivery_to",
"delivery_account_id",
"message",
"continuation_json",
"doctor_hint",
"stats_json",
"updated_at_ms",
])
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
);
if (!row) {
return { kind: "missing" };
}
const sentinel = decodeRestartSentinelRow(row);
return sentinel ? { kind: "valid", sentinel } : { kind: "invalid", revision: row.updated_at_ms };
}
function requireValidPayload(payload: RestartSentinelPayload): RestartSentinelPayload {
const parsed = parseRestartSentinelPayload(payload);
if (!parsed) {
throw new TypeError("Invalid restart sentinel payload");
}
return parsed;
}
function nextRevision(currentRevision: number | null): number {
if (currentRevision !== null && !Number.isSafeInteger(currentRevision)) {
throw new Error("Restart sentinel revision is outside the safe integer range");
}
// Same-millisecond replacements still need distinct revisions, or a stale
// consumer could delete the newer singleton row after delivering the old one.
const revision = Math.max(Date.now(), currentRevision === null ? 0 : currentRevision + 1);
if (!Number.isSafeInteger(revision)) {
throw new Error("Restart sentinel revision exhausted the safe integer range");
}
return revision;
}
function readRestartSentinelRevisionFloorSync(db: DatabaseSync): number | null {
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
const row = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select("updated_at_ms")
.where("sentinel_key", "=", RESTART_SENTINEL_REVISION_FLOOR_KEY),
);
if (!row) {
return null;
}
if (!Number.isSafeInteger(row.updated_at_ms)) {
throw new Error("Restart sentinel revision floor is outside the safe integer range");
}
return row.updated_at_ms;
}
function maxRevision(left: number | null, right: number | null): number | null {
if (left === null) {
return right;
}
if (right === null) {
return left;
}
return Math.max(left, right);
}
function buildRestartSentinelRow(
payload: RestartSentinelPayload,
revision: number,
sentinelKey = RESTART_SENTINEL_KEY,
) {
return {
sentinel_key: sentinelKey,
version: 1,
kind: payload.kind,
status: payload.status,
ts: payload.ts,
session_key: payload.sessionKey ?? null,
thread_id: payload.threadId ?? null,
delivery_channel: payload.deliveryContext?.channel ?? null,
delivery_to: payload.deliveryContext?.to ?? null,
delivery_account_id: payload.deliveryContext?.accountId ?? null,
message: payload.message ?? null,
continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null,
doctor_hint: payload.doctorHint ?? null,
stats_json: payload.stats ? JSON.stringify(payload.stats) : null,
// Debug shadow only. Reads reconstruct exclusively from typed columns above.
payload_json: JSON.stringify(payload),
updated_at_ms: revision,
};
}
function upsertRestartSentinelRowSync(
db: DatabaseSync,
row: ReturnType<typeof buildRestartSentinelRow>,
): void {
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.insertInto("gateway_restart_sentinel")
.values(row)
.onConflict((conflict) =>
conflict.column("sentinel_key").doUpdateSet({
version: (eb) => eb.ref("excluded.version"),
kind: (eb) => eb.ref("excluded.kind"),
status: (eb) => eb.ref("excluded.status"),
ts: (eb) => eb.ref("excluded.ts"),
session_key: (eb) => eb.ref("excluded.session_key"),
thread_id: (eb) => eb.ref("excluded.thread_id"),
delivery_channel: (eb) => eb.ref("excluded.delivery_channel"),
delivery_to: (eb) => eb.ref("excluded.delivery_to"),
delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"),
message: (eb) => eb.ref("excluded.message"),
continuation_json: (eb) => eb.ref("excluded.continuation_json"),
doctor_hint: (eb) => eb.ref("excluded.doctor_hint"),
stats_json: (eb) => eb.ref("excluded.stats_json"),
payload_json: (eb) => eb.ref("excluded.payload_json"),
updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"),
}),
),
);
}
function advanceRestartSentinelRevisionFloorSync(db: DatabaseSync, revision: number): void {
// `current` is deleted after durable delivery. The reserved row survives that
// clear so a later same-millisecond write cannot reuse an idempotency revision.
const payload: RestartSentinelPayload = { kind: "restart", status: "skipped", ts: revision };
upsertRestartSentinelRowSync(
db,
buildRestartSentinelRow(payload, revision, RESTART_SENTINEL_REVISION_FLOOR_KEY),
);
}
export function writeRestartSentinelRowSync(
db: DatabaseSync,
rawPayload: RestartSentinelPayload,
): RestartSentinel {
const payload = requireValidPayload(rawPayload);
const current = readRestartSentinelRowSync(db);
const currentRevision =
current.kind === "missing"
? null
: current.kind === "valid"
? current.sentinel.revision
: current.revision;
const revision = nextRevision(
maxRevision(currentRevision, readRestartSentinelRevisionFloorSync(db)),
);
const row = buildRestartSentinelRow(payload, revision);
upsertRestartSentinelRowSync(db, row);
advanceRestartSentinelRevisionFloorSync(db, revision);
return { version: 1, payload, revision };
}
export function writeRestartSentinelRowIfRevisionSync(
db: DatabaseSync,
rawPayload: RestartSentinelPayload,
expectedRevision: number,
): RestartSentinel | null {
const current = readRestartSentinelRowSync(db);
if (current.kind !== "valid" || current.sentinel.revision !== expectedRevision) {
return null;
}
const payload = requireValidPayload(rawPayload);
const revision = nextRevision(
maxRevision(expectedRevision, readRestartSentinelRevisionFloorSync(db)),
);
const row = buildRestartSentinelRow(payload, revision);
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
const result = executeSqliteQuerySync(
db,
stateDb
.updateTable("gateway_restart_sentinel")
.set(row)
.where("sentinel_key", "=", RESTART_SENTINEL_KEY)
.where("updated_at_ms", "=", expectedRevision),
);
if (result.numAffectedRows !== 1n) {
return null;
}
advanceRestartSentinelRevisionFloorSync(db, revision);
return { version: 1, payload, revision };
}
export function deleteRestartSentinelRowSync(db: DatabaseSync, expectedRevision?: number): boolean {
const current = readRestartSentinelRowSync(db);
if (current.kind === "missing") {
return false;
}
const currentRevision = current.kind === "valid" ? current.sentinel.revision : current.revision;
if (expectedRevision !== undefined && currentRevision !== expectedRevision) {
return false;
}
if (!Number.isSafeInteger(currentRevision)) {
throw new Error("Restart sentinel revision is outside the safe integer range");
}
advanceRestartSentinelRevisionFloorSync(
db,
maxRevision(currentRevision, readRestartSentinelRevisionFloorSync(db)) ?? currentRevision,
);
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
let query = stateDb
.deleteFrom("gateway_restart_sentinel")
.where("sentinel_key", "=", RESTART_SENTINEL_KEY);
if (expectedRevision !== undefined) {
query = query.where("updated_at_ms", "=", expectedRevision);
}
if (executeSqliteQuerySync(db, query).numAffectedRows !== 1n) {
// The outer write transaction owns both rows; fail closed so its rollback
// cannot leave a floor for a current row this call did not consume.
throw new Error("Restart sentinel changed during guarded delete");
}
return true;
}
+176 -121
View File
@@ -45,6 +45,7 @@ import {
import {
buildRestartSuccessContinuation,
clearRestartSentinel,
clearRestartSentinelIfRevision,
finalizeUpdateRestartSentinelRunningVersion,
formatDoctorNonInteractiveHint,
formatRestartSentinelMessage,
@@ -87,34 +88,51 @@ function readSentinelRow() {
db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select(["sentinel_key", "version", "kind", "status", "payload_json"])
.select(["sentinel_key", "version", "kind", "status", "payload_json", "updated_at_ms"])
.where("sentinel_key", "=", "current"),
);
}
function insertSentinelRow(values: { version?: number; payloadJson: string }) {
function readSentinelRevisionFloor() {
const { db } = openOpenClawStateDatabase();
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
return executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select("updated_at_ms")
.where("sentinel_key", "=", "revision-floor"),
)?.updated_at_ms;
}
function deleteSentinelRevisionFloor() {
const { db } = openOpenClawStateDatabase();
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb.insertInto("gateway_restart_sentinel").values({
sentinel_key: "current",
version: values.version ?? 1,
kind: "update",
status: "ok",
ts: Date.now(),
session_key: null,
thread_id: null,
delivery_channel: null,
delivery_to: null,
delivery_account_id: null,
message: null,
continuation_json: null,
doctor_hint: null,
stats_json: null,
payload_json: values.payloadJson,
updated_at_ms: Date.now(),
}),
stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "revision-floor"),
);
}
function updateSentinelRow(
values: Partial<{
version: number;
kind: string;
status: string;
continuation_json: string | null;
stats_json: string | null;
payload_json: string;
updated_at_ms: number;
}>,
) {
const { db } = openOpenClawStateDatabase();
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.updateTable("gateway_restart_sentinel")
.set(values)
.where("sentinel_key", "=", "current"),
);
}
@@ -147,7 +165,28 @@ describe("restart sentinel", () => {
});
});
it("imports a legacy file sentinel into sqlite once", async () => {
it("canonicalizes nullable top-level fields and empty delivery context", async () => {
await withRestartSentinelStateDir(async () => {
const written = await writeRestartSentinel({
kind: "restart",
status: "ok",
ts: 1,
deliveryContext: {},
message: null,
continuation: null,
doctorHint: null,
stats: null,
});
expect(written.payload).toEqual({ kind: "restart", status: "ok", ts: 1 });
await expect(readRestartSentinel()).resolves.toEqual(written);
expect(readSentinelRow()?.payload_json).toBe(
JSON.stringify({ kind: "restart", status: "ok", ts: 1 }),
);
});
});
it("ignores legacy files without mutating them", async () => {
await withRestartSentinelStateDir(async () => {
const payload = {
kind: "update" as const,
@@ -161,100 +200,132 @@ describe("restart sentinel", () => {
},
};
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
await fs.writeFile(legacyPath, `${JSON.stringify({ version: 1, payload })}\n`, "utf-8");
await expect(hasRestartSentinel()).resolves.toBe(true);
expect(readSentinelRow()).toMatchObject({
sentinel_key: "current",
version: 1,
kind: "update",
status: "skipped",
payload_json: JSON.stringify(payload),
});
await expect(fs.access(legacyPath)).rejects.toThrow();
await expect(readRestartSentinel()).resolves.toEqual({ version: 1, payload });
});
});
it("does not replay a legacy file superseded by a sqlite sentinel", async () => {
await withRestartSentinelStateDir(async () => {
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
await fs.writeFile(
legacyPath,
`${JSON.stringify({
version: 1,
payload: {
kind: "update",
status: "ok",
ts: 1,
message: "stale legacy sentinel",
},
})}\n`,
"utf-8",
);
await writeRestartSentinel({
kind: "restart",
status: "ok",
ts: 2,
message: "current sqlite sentinel",
});
await expect(fs.access(legacyPath)).rejects.toThrow();
await clearRestartSentinel();
const legacyContents = `${JSON.stringify({ version: 1, payload })}\n`;
await fs.writeFile(legacyPath, legacyContents, "utf-8");
await expect(hasRestartSentinel()).resolves.toBe(false);
await expect(readRestartSentinel()).resolves.toBeNull();
await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 });
await clearRestartSentinel();
await expect(fs.readFile(legacyPath, "utf-8")).resolves.toBe(legacyContents);
});
});
it("drops invalid sentinel payloads", async () => {
it("reconstructs typed columns when payload_json is corrupt", async () => {
await withRestartSentinelStateDir(async () => {
insertSentinelRow({ payloadJson: "not-json" });
const payload = {
kind: "update" as const,
status: "skipped" as const,
ts: 42,
sessionKey: "agent:main:webchat:dm:user-123",
deliveryContext: { channel: "webchat", to: "user-123", accountId: "default" },
threadId: "thread-1",
message: "typed state",
continuation: { kind: "agentTurn" as const, message: "continue" },
doctorHint: "run doctor",
stats: { mode: "npm", reason: "pending" },
};
const written = await writeRestartSentinel(payload);
updateSentinelRow({ payload_json: "not-json" });
const read = await readRestartSentinel();
expect(read).toBeNull();
expect(readSentinelRow()).toBeUndefined();
await expect(readRestartSentinel()).resolves.toEqual(written);
});
});
it("drops structurally invalid sentinel payloads", async () => {
it("leaves malformed typed rows in place and reports them as unreadable", async () => {
await withRestartSentinelStateDir(async () => {
insertSentinelRow({ version: 2, payloadJson: JSON.stringify(null) });
await writeRestartSentinel({ kind: "update", status: "ok", ts: 1 });
updateSentinelRow({ kind: "not-a-kind", payload_json: "{}" });
await expect(readRestartSentinel()).resolves.toBeNull();
expect(readSentinelRow()).toBeUndefined();
await expect(hasRestartSentinel()).resolves.toBe(false);
expect(readSentinelRow()).toMatchObject({ kind: "not-a-kind", payload_json: "{}" });
expect(mockWarn).toHaveBeenCalledWith("Ignoring invalid typed restart sentinel row");
});
});
it("keeps old config restart sentinels readable without restart-required stats", async () => {
it("rejects malformed typed JSON columns even when the shadow payload is valid", async () => {
await withRestartSentinelStateDir(async () => {
const filePath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
const payload = {
kind: "config-patch" as const,
status: "ok" as const,
ts: Date.now(),
message: "Config updated successfully",
stats: { mode: "config.patch" },
};
await fs.mkdir(path.dirname(filePath), { recursive: true });
await fs.writeFile(filePath, JSON.stringify({ version: 1, payload }, null, 2), "utf-8");
const payload = { kind: "update" as const, status: "ok" as const, ts: 1 };
await writeRestartSentinel(payload);
updateSentinelRow({
continuation_json: JSON.stringify({ kind: "agentTurn", message: 42 }),
payload_json: JSON.stringify(payload),
});
const read = await readRestartSentinel();
await expect(readRestartSentinel()).resolves.toBeNull();
await expect(hasRestartSentinel()).resolves.toBe(false);
});
});
expect(read?.payload).toEqual(payload);
if (!read) {
throw new Error("Expected old restart sentinel to be readable");
it("keeps revisions strictly monotonic within the same millisecond", async () => {
await withRestartSentinelStateDir(async () => {
const now = vi.spyOn(Date, "now").mockReturnValue(1000);
try {
const first = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 });
const second = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 });
expect(second.revision).toBe(first.revision + 1);
expect(readSentinelRow()?.updated_at_ms).toBe(second.revision);
} finally {
now.mockRestore();
}
expect(summarizeRestartSentinel(read.payload)).toBe(
"Gateway restart config-patch ok (config.patch)",
);
expect(formatRestartSentinelMessage(read.payload)).toBe(
["Gateway restart config-patch ok (config.patch)", "Config updated successfully"].join(
"\n",
),
);
});
});
it("upgrades pre-floor rows before unconditional and guarded clears", async () => {
await withRestartSentinelStateDir(async () => {
const now = vi.spyOn(Date, "now").mockReturnValue(1000);
try {
const first = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 });
deleteSentinelRevisionFloor();
expect(readSentinelRevisionFloor()).toBeUndefined();
await expect(clearRestartSentinel()).resolves.toBe(true);
await expect(readRestartSentinel()).resolves.toBeNull();
await expect(hasRestartSentinel()).resolves.toBe(false);
expect(readSentinelRevisionFloor()).toBe(first.revision);
now.mockReturnValue(500);
const second = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 });
expect(second.revision).toBe(first.revision + 1);
deleteSentinelRevisionFloor();
await expect(clearRestartSentinelIfRevision(second.revision + 1)).resolves.toBe(false);
expect(readSentinelRevisionFloor()).toBeUndefined();
await expect(readRestartSentinel()).resolves.toEqual(second);
await expect(clearRestartSentinelIfRevision(second.revision)).resolves.toBe(true);
expect(readSentinelRevisionFloor()).toBe(second.revision);
const third = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 3 });
expect(third.revision).toBe(second.revision + 1);
await expect(clearRestartSentinelIfRevision(third.revision)).resolves.toBe(true);
deleteSentinelRevisionFloor();
await expect(clearRestartSentinel()).resolves.toBe(false);
expect(readSentinelRevisionFloor()).toBeUndefined();
} finally {
now.mockRestore();
}
});
});
it("does not let stale deletes remove a newer sentinel", async () => {
await withRestartSentinelStateDir(async () => {
const first = await writeRestartSentinel({
kind: "restart",
status: "ok",
ts: 1,
message: "old",
});
const newer = await writeRestartSentinel({
kind: "restart",
status: "ok",
ts: 2,
message: "new",
});
await expect(clearRestartSentinelIfRevision(first.revision)).resolves.toBe(false);
await expect(readRestartSentinel()).resolves.toEqual(newer);
});
});
@@ -422,7 +493,7 @@ describe("restart sentinel", () => {
await finalizeUpdateRestartSentinelRunningVersion("actual-version");
await expect(readRestartSentinel()).resolves.toEqual({
await expect(readRestartSentinel()).resolves.toMatchObject({
version: 1,
payload: {
kind: "update",
@@ -453,7 +524,7 @@ describe("restart sentinel", () => {
await expect(
finalizeUpdateRestartSentinelRunningVersion("actual-version"),
).resolves.toBeNull();
await expect(readRestartSentinel()).resolves.toEqual({
await expect(readRestartSentinel()).resolves.toMatchObject({
version: 1,
payload: {
kind: "update",
@@ -481,7 +552,7 @@ describe("restart sentinel", () => {
await markUpdateRestartSentinelFailure("restart-unhealthy");
await expect(readRestartSentinel()).resolves.toEqual({
await expect(readRestartSentinel()).resolves.toMatchObject({
version: 1,
payload: {
kind: "update",
@@ -497,18 +568,16 @@ describe("restart sentinel", () => {
});
describe("restart sentinel error visibility", () => {
it("logs a warning when clearRestartSentinel DB write fails", async () => {
mockThrowWrite.mockImplementationOnce(() => {
throw new Error("SQLITE_IOERR: disk I/O error");
});
it("throws when clearRestartSentinel cannot durably delete the row", async () => {
await withRestartSentinelStateDir(async () => {
await expect(clearRestartSentinel()).resolves.toBeUndefined();
const written = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 });
mockThrowWrite.mockImplementationOnce(() => {
throw new Error("SQLITE_IOERR: disk I/O error");
});
expect(mockWarn).toHaveBeenCalledTimes(1);
expect(mockWarn).toHaveBeenCalledWith(
"Failed to clear restart sentinel: SQLITE_IOERR: disk I/O error",
);
await expect(clearRestartSentinel()).rejects.toThrow("SQLITE_IOERR: disk I/O error");
expect(mockWarn).not.toHaveBeenCalled();
await expect(readRestartSentinel()).resolves.toEqual(written);
});
});
@@ -541,20 +610,6 @@ describe("restart sentinel error visibility", () => {
);
});
});
it("logs a warning when the legacy sentinel path cannot be removed", async () => {
await withRestartSentinelStateDir(async () => {
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
await fs.mkdir(legacyPath);
await expect(clearRestartSentinel()).resolves.toBeUndefined();
expect(mockWarn).toHaveBeenCalledTimes(1);
expect(mockWarn).toHaveBeenCalledWith(
expect.stringMatching(/^Failed to remove legacy restart sentinel: .+/),
);
});
});
});
describe("restart success continuation", () => {
+54 -217
View File
@@ -1,12 +1,8 @@
// Persists restart sentinel state that coordinates deferred restarts.
import { readFile, rm } from "node:fs/promises";
import path from "node:path";
import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce";
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { formatCliCommand } from "../cli/command-format.js";
import { resolveStateDir } from "../config/paths.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
openOpenClawStateDatabase,
runOpenClawStateWriteTransaction,
@@ -14,77 +10,22 @@ import {
import { resolveRuntimeServiceVersion } from "../version.js";
import { formatErrorMessage } from "./errors.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
deleteRestartSentinelRowSync,
readRestartSentinelRowSync,
writeRestartSentinelRowIfRevisionSync,
writeRestartSentinelRowSync,
type RestartSentinel,
type RestartSentinelContinuation,
type RestartSentinelPayload,
} from "./restart-sentinel-store.js";
export type {
RestartSentinelContinuation,
RestartSentinelPayload,
} from "./restart-sentinel-store.js";
const sentinelLog = createSubsystemLogger("restart-sentinel");
type RestartSentinelLog = {
stdoutTail?: string | null;
stderrTail?: string | null;
exitCode?: number | null;
};
type RestartSentinelStep = {
name: string;
command: string;
cwd?: string | null;
durationMs?: number | null;
log?: RestartSentinelLog | null;
};
type RestartSentinelStats = {
mode?: string;
root?: string;
requiresRestart?: boolean;
handoffId?: string;
before?: Record<string, unknown> | null;
after?: Record<string, unknown> | null;
steps?: RestartSentinelStep[];
reason?: string | null;
durationMs?: number | null;
};
export type RestartSentinelContinuation =
| {
kind: "systemEvent";
text: string;
}
| {
kind: "agentTurn";
message: string;
};
export type RestartSentinelPayload = {
kind: "config-apply" | "config-auto-recovery" | "config-patch" | "update" | "restart";
status: "ok" | "error" | "skipped";
ts: number;
sessionKey?: string;
/** Delivery context captured at restart time to ensure channel routing survives restart. */
deliveryContext?: {
channel?: string;
to?: string;
accountId?: string;
};
/** Thread ID for reply threading (e.g., Slack thread_ts). */
threadId?: string;
message?: string | null;
continuation?: RestartSentinelContinuation | null;
doctorHint?: string | null;
stats?: RestartSentinelStats | null;
};
type RestartSentinel = {
version: 1;
payload: RestartSentinelPayload;
};
const RESTART_SENTINEL_KEY = "current";
const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json";
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
export function formatDoctorNonInteractiveHint(
env: Record<string, string | undefined> = process.env as Record<string, string | undefined>,
): string {
@@ -97,57 +38,12 @@ export function formatDoctorNonInteractiveHint(
export async function writeRestartSentinel(
payload: RestartSentinelPayload,
env: NodeJS.ProcessEnv = process.env,
): Promise<void> {
const updatedAtMs = Date.now();
runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.insertInto("gateway_restart_sentinel")
.values({
sentinel_key: RESTART_SENTINEL_KEY,
version: 1,
kind: payload.kind,
status: payload.status,
ts: payload.ts,
session_key: payload.sessionKey ?? null,
thread_id: payload.threadId ?? null,
delivery_channel: payload.deliveryContext?.channel ?? null,
delivery_to: payload.deliveryContext?.to ?? null,
delivery_account_id: payload.deliveryContext?.accountId ?? null,
message: payload.message ?? null,
continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null,
doctor_hint: payload.doctorHint ?? null,
stats_json: payload.stats ? JSON.stringify(payload.stats) : null,
payload_json: JSON.stringify(payload),
updated_at_ms: updatedAtMs,
})
.onConflict((conflict) =>
conflict.column("sentinel_key").doUpdateSet({
version: (eb) => eb.ref("excluded.version"),
kind: (eb) => eb.ref("excluded.kind"),
status: (eb) => eb.ref("excluded.status"),
ts: (eb) => eb.ref("excluded.ts"),
session_key: (eb) => eb.ref("excluded.session_key"),
thread_id: (eb) => eb.ref("excluded.thread_id"),
delivery_channel: (eb) => eb.ref("excluded.delivery_channel"),
delivery_to: (eb) => eb.ref("excluded.delivery_to"),
delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"),
message: (eb) => eb.ref("excluded.message"),
continuation_json: (eb) => eb.ref("excluded.continuation_json"),
doctor_hint: (eb) => eb.ref("excluded.doctor_hint"),
stats_json: (eb) => eb.ref("excluded.stats_json"),
payload_json: (eb) => eb.ref("excluded.payload_json"),
updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"),
}),
),
);
},
): Promise<RestartSentinel> {
return runOpenClawStateWriteTransaction(
({ db }) => writeRestartSentinelRowSync(db, payload),
{ env },
{ operationLabel: "restart-sentinel.write" },
);
await removeLegacyRestartSentinel(env);
}
function cloneRestartSentinelPayload(payload: RestartSentinelPayload): RestartSentinelPayload {
@@ -158,19 +54,20 @@ async function rewriteRestartSentinel(
rewrite: (payload: RestartSentinelPayload) => RestartSentinelPayload | null,
env: NodeJS.ProcessEnv = process.env,
): Promise<RestartSentinel | null> {
const current = await readRestartSentinel(env);
if (!current) {
return null;
}
const nextPayload = rewrite(cloneRestartSentinelPayload(current.payload));
if (!nextPayload) {
return null;
}
await writeRestartSentinel(nextPayload, env);
return {
version: 1,
payload: nextPayload,
};
return runOpenClawStateWriteTransaction(
({ db }) => {
const current = readRestartSentinelRowSync(db);
if (current.kind !== "valid") {
return null;
}
const nextPayload = rewrite(cloneRestartSentinelPayload(current.sentinel.payload));
return nextPayload
? writeRestartSentinelRowIfRevisionSync(db, nextPayload, current.sentinel.revision)
: null;
},
{ env },
{ operationLabel: "restart-sentinel.rewrite-current" },
);
}
export async function finalizeUpdateRestartSentinelRunningVersion(
@@ -215,60 +112,23 @@ export async function markUpdateRestartSentinelFailure(
}, env);
}
export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<void> {
try {
runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.deleteFrom("gateway_restart_sentinel")
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
);
},
{ env },
);
} catch (err) {
// Clearing the sentinel is best-effort during shutdown/cleanup, but a
// failure here may leave the gateway believing a restart is still pending.
sentinelLog.warn(`Failed to clear restart sentinel: ${formatErrorMessage(err)}`);
}
await removeLegacyRestartSentinel(env);
export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
return runOpenClawStateWriteTransaction(
({ db }) => deleteRestartSentinelRowSync(db),
{ env },
{ operationLabel: "restart-sentinel.clear" },
);
}
function resolveLegacyRestartSentinelPath(env: NodeJS.ProcessEnv): string {
return path.join(resolveStateDir(env), LEGACY_RESTART_SENTINEL_FILENAME);
}
async function removeLegacyRestartSentinel(env: NodeJS.ProcessEnv): Promise<void> {
try {
await rm(resolveLegacyRestartSentinelPath(env), { force: true });
} catch (err) {
// Legacy cleanup must not block the canonical SQLite operation, but a
// failed removal can replay stale restart state after the database clears.
sentinelLog.warn(`Failed to remove legacy restart sentinel: ${formatErrorMessage(err)}`);
}
}
async function importLegacyRestartSentinel(
export async function clearRestartSentinelIfRevision(
expectedRevision: number,
env: NodeJS.ProcessEnv = process.env,
): Promise<RestartSentinel | null> {
const legacyPath = resolveLegacyRestartSentinelPath(env);
let parsed: unknown;
try {
parsed = JSON.parse(await readFile(legacyPath, "utf-8")) as unknown;
} catch {
return null;
}
if (!isPlainRecord(parsed) || parsed.version !== 1 || !isPlainRecord(parsed.payload)) {
await removeLegacyRestartSentinel(env);
return null;
}
const payload = parsed.payload as RestartSentinelPayload;
await writeRestartSentinel(payload, env);
await removeLegacyRestartSentinel(env);
return { version: 1, payload };
): Promise<boolean> {
return runOpenClawStateWriteTransaction(
({ db }) => deleteRestartSentinelRowSync(db, expectedRevision),
{ env },
{ operationLabel: "restart-sentinel.clear-if-revision" },
);
}
export function buildRestartSuccessContinuation(params: {
@@ -287,29 +147,12 @@ export async function readRestartSentinel(
): Promise<RestartSentinel | null> {
try {
const database = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(database.db);
const row = executeSqliteQueryTakeFirstSync(
database.db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select(["version", "payload_json"])
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
);
if (!row) {
return await importLegacyRestartSentinel(env);
}
let payload: RestartSentinelPayload | undefined;
try {
payload = JSON.parse(row.payload_json) as RestartSentinelPayload | undefined;
} catch {
await clearRestartSentinel(env);
const current = readRestartSentinelRowSync(database.db);
if (current.kind === "invalid") {
sentinelLog.warn("Ignoring invalid typed restart sentinel row");
return null;
}
if (row.version !== 1 || !payload) {
await clearRestartSentinel(env);
return null;
}
return { version: 1, payload };
return current.kind === "valid" ? current.sentinel : null;
} catch (err) {
sentinelLog.warn(`Failed to read restart sentinel: ${formatErrorMessage(err)}`);
return null;
@@ -319,18 +162,12 @@ export async function readRestartSentinel(
export async function hasRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
try {
const database = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(database.db);
const row = executeSqliteQueryTakeFirstSync(
database.db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select("sentinel_key")
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
);
if (row) {
return true;
const current = readRestartSentinelRowSync(database.db);
if (current.kind === "invalid") {
sentinelLog.warn("Ignoring invalid typed restart sentinel row");
return false;
}
return Boolean(await importLegacyRestartSentinel(env));
return current.kind === "valid";
} catch (err) {
sentinelLog.warn(`Failed to check restart sentinel: ${formatErrorMessage(err)}`);
return false;
+6 -1
View File
@@ -11,6 +11,7 @@ import {
moveDeliveryQueueEntryToFailed,
updateDeliveryQueueEntry,
upsertDeliveryQueueEntry,
type DeliveryQueueCompletionRetention,
type DeliveryQueueRowMetadata,
} from "./delivery-queue-sqlite.js";
import { generateSecureUuid } from "./secure-random.js";
@@ -28,6 +29,8 @@ type SessionDeliveryContext = {
type SessionDeliveryRetryPolicy = {
maxRetries?: number;
/** Retain terminal ownership when the durable producer can replay forever. */
completionRetention?: DeliveryQueueCompletionRetention;
};
export type SessionDeliveryRoute = {
@@ -139,7 +142,9 @@ export async function enqueueSessionDelivery(
entry,
metadata: queuedSessionDeliveryMetadata(entry),
stateDir,
reviveFailedOrCorruptPending: Boolean(params.idempotencyKey),
...(params.completionRetention === "permanent"
? { insertOnly: true }
: { reviveFailedOrCorruptPending: Boolean(params.idempotencyKey) }),
});
return id;
}
@@ -129,6 +129,24 @@ describe("session-delivery queue storage", () => {
});
});
it("never revives a failed permanent producer intent", async () => {
await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => {
const payload = {
kind: "systemEvent" as const,
sessionKey: "agent:main:main",
text: "restart complete",
idempotencyKey: "restart:permanent-failed",
completionRetention: "permanent" as const,
};
const id = await enqueueSessionDelivery(payload, tempDir);
await moveSessionDeliveryToFailed(id, tempDir);
expect(await enqueueSessionDelivery(payload, tempDir)).toBe(id);
expect(readSessionQueueStatus(tempDir, id)).toBe("failed");
expect(await loadPendingSessionDeliveries(tempDir)).toEqual([]);
});
});
it("reports a completed conflict after acknowledgement", async () => {
await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => {
const payload = {
@@ -355,4 +373,22 @@ describe("session-delivery queue storage", () => {
expect(readSessionQueueStatus(tempDir, id)).toBe("completed");
});
});
it("retains a permanent completion receipt", async () => {
await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => {
const payload = {
kind: "systemEvent" as const,
sessionKey: "agent:main:main",
text: "restart complete",
idempotencyKey: "restart:permanent-completed",
completionRetention: "permanent" as const,
};
const id = await enqueueSessionDelivery(payload, tempDir);
await settleSessionDelivery(id, tempDir);
expect(await enqueueSessionDelivery(payload, tempDir)).toBe(id);
expect(readSessionQueueStatus(tempDir, id)).toBe("completed");
expect(await loadPendingSessionDeliveries(tempDir)).toEqual([]);
});
});
});
+49 -2
View File
@@ -79,6 +79,10 @@ import {
detectLegacyRescuePending,
discardLegacyRescuePending,
} from "./state-migrations.rescue-pending.js";
import {
detectLegacyRestartSentinel,
migrateLegacyRestartSentinel,
} from "./state-migrations.restart-sentinel.js";
import {
migrateLegacyConfigHealth,
migrateLegacyCurrentConversationBindings,
@@ -416,6 +420,7 @@ export async function detectLegacyStateMigrations(params: {
stateDir,
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations,
});
const restartSentinel = detectLegacyRestartSentinel({ stateDir });
const workspace = detectLegacyWorkspaceState({
cfg: params.cfg,
stateDir,
@@ -598,6 +603,9 @@ export async function detectLegacyStateMigrations(params: {
if (mcpOauth.hasLegacy) {
preview.push("- MCP OAuth credentials: legacy JSON → shared SQLite state");
}
if (restartSentinel.hasLegacy) {
preview.push("- Restart sentinel: legacy JSON → shared SQLite state");
}
if (workspace.hasLegacy) {
preview.push("- Workspace setup and attestations: legacy files → shared SQLite state");
}
@@ -701,6 +709,7 @@ export async function detectLegacyStateMigrations(params: {
managedOutgoingImages,
apns,
mcpOauth,
restartSentinel,
workspace,
webPush,
nodeHost,
@@ -909,6 +918,11 @@ export async function runLegacyStateMigrations(params: {
env,
stateDir: detected.stateDir,
});
const restartSentinel = await migrateLegacyRestartSentinel({
detected: detected.restartSentinel,
env,
stateDir: detected.stateDir,
});
const workspace = await migrateLegacyWorkspaceState({
detected: detected.workspace,
env,
@@ -968,6 +982,7 @@ export async function runLegacyStateMigrations(params: {
managedOutgoingImages,
apns,
mcpOauth,
restartSentinel,
workspace,
webPush,
nodeHost,
@@ -993,6 +1008,7 @@ export async function runLegacyStateMigrations(params: {
...managedOutgoingImages.changes,
...apns.changes,
...mcpOauth.changes,
...restartSentinel.changes,
...workspace.changes,
...webPush.changes,
...nodeHost.changes,
@@ -1025,6 +1041,7 @@ export async function runLegacyStateMigrations(params: {
...managedOutgoingImages.warnings,
...apns.warnings,
...mcpOauth.warnings,
...restartSentinel.warnings,
...workspace.warnings,
...webPush.warnings,
...nodeHost.warnings,
@@ -1187,6 +1204,11 @@ export async function autoMigrateLegacyState(params: {
detected: detected.currentConversationBindings,
stateDir: detected.stateDir,
});
const restartSentinel = await migrateLegacyRestartSentinel({
detected: detected.restartSentinel,
env,
stateDir: detected.stateDir,
});
const channelPairing = migrateLegacyChannelPairingState({
detected: detected.channelPairing,
env: { ...env, OPENCLAW_STATE_DIR: detected.stateDir },
@@ -1214,6 +1236,7 @@ export async function autoMigrateLegacyState(params: {
...configHealth.changes,
...pluginBindingApprovals.changes,
...currentConversationBindings.changes,
...restartSentinel.changes,
...channelPairing.changes,
...preSessionChannelPlans.changes,
...pluginPlans.changes,
@@ -1234,11 +1257,19 @@ export async function autoMigrateLegacyState(params: {
...configHealth.warnings,
...pluginBindingApprovals.warnings,
...currentConversationBindings.warnings,
...restartSentinel.warnings,
...channelPairing.warnings,
...preSessionChannelPlans.warnings,
...pluginPlans.warnings,
];
const noticeSources = [stateDirResult, detected, pluginInstallIndex, updateCheck, pluginPlans];
const noticeSources = [
stateDirResult,
detected,
pluginInstallIndex,
updateCheck,
restartSentinel,
pluginPlans,
];
const notices = mergeNotices(noticeSources);
logMigrationResults(changes, warnings, notices);
return {
@@ -1257,6 +1288,7 @@ export async function autoMigrateLegacyState(params: {
configHealth.changes.length > 0 ||
pluginBindingApprovals.changes.length > 0 ||
currentConversationBindings.changes.length > 0 ||
restartSentinel.changes.length > 0 ||
channelPairing.changes.length > 0 ||
preSessionChannelPlans.changes.length > 0 ||
pluginPlans.changes.length > 0,
@@ -1282,6 +1314,7 @@ export async function autoMigrateLegacyState(params: {
!detected.configHealth.hasLegacy &&
!detected.pluginBindingApprovals.hasLegacy &&
!detected.currentConversationBindings.hasLegacy &&
!detected.restartSentinel?.hasLegacy &&
!detected.workspace.hasLegacy &&
!detected.channelPairing.hasLegacy
) {
@@ -1350,6 +1383,11 @@ export async function autoMigrateLegacyState(params: {
detected: detected.currentConversationBindings,
stateDir: detected.stateDir,
});
const restartSentinel = await migrateLegacyRestartSentinel({
detected: detected.restartSentinel,
env,
stateDir: detected.stateDir,
});
const channelPairing = migrateLegacyChannelPairingState({
detected: detected.channelPairing,
env: { ...env, OPENCLAW_STATE_DIR: detected.stateDir },
@@ -1390,6 +1428,7 @@ export async function autoMigrateLegacyState(params: {
...configHealth.changes,
...pluginBindingApprovals.changes,
...currentConversationBindings.changes,
...restartSentinel.changes,
...channelPairing.changes,
...preSessionChannelPlans.changes,
...pluginPlans.changes,
@@ -1414,6 +1453,7 @@ export async function autoMigrateLegacyState(params: {
...configHealth.warnings,
...pluginBindingApprovals.warnings,
...currentConversationBindings.warnings,
...restartSentinel.warnings,
...channelPairing.warnings,
...preSessionChannelPlans.warnings,
...pluginPlans.warnings,
@@ -1422,7 +1462,14 @@ export async function autoMigrateLegacyState(params: {
...agentDir.warnings,
...channelPlans.warnings,
];
const noticeSources = [stateDirResult, detected, pluginInstallIndex, updateCheck, pluginPlans];
const noticeSources = [
stateDirResult,
detected,
pluginInstallIndex,
updateCheck,
restartSentinel,
pluginPlans,
];
const notices = mergeNotices(noticeSources);
logMigrationResults(changes, warnings, notices);
@@ -0,0 +1,365 @@
// Covers safe startup/Doctor import of the retired restart-sentinel JSON file.
import fs from "node:fs";
import fsp from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
} from "../state/openclaw-state-db.js";
import { acquireGatewayLock } from "./gateway-lock.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
import {
clearRestartSentinel,
readRestartSentinel,
writeRestartSentinel,
type RestartSentinelPayload,
} from "./restart-sentinel.js";
import {
detectLegacyRestartSentinel,
migrateLegacyRestartSentinel,
} from "./state-migrations.restart-sentinel.js";
type MigrationDatabase = Pick<
OpenClawStateKyselyDatabase,
"gateway_restart_sentinel" | "migration_sources"
>;
describe("legacy restart sentinel migration", () => {
const tempDirs = useAutoCleanupTempDirTracker((cleanup) => {
afterEach(() => {
closeOpenClawStateDatabaseForTest();
cleanup();
});
});
function useStateDir(): { env: NodeJS.ProcessEnv; stateDir: string } {
const stateDir = tempDirs.make("openclaw-restart-sentinel-migration-");
return { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, stateDir };
}
function payload(ts = 123): RestartSentinelPayload {
return {
kind: "update",
status: "ok",
ts,
sessionKey: "agent:main:main",
deliveryContext: { channel: "test", to: "target", accountId: "default" },
threadId: "thread-1",
message: "Update completed",
continuation: { kind: "agentTurn", message: "Continue after restart" },
doctorHint: "Run Doctor",
stats: {
mode: "managed",
handoffId: "handoff-1",
requiresRestart: true,
before: { version: "old" },
after: { version: "new" },
steps: [
{
name: "install",
command: "package-manager update",
durationMs: 10,
log: { stdoutTail: "done", stderrTail: null, exitCode: 0 },
},
],
},
};
}
async function writeLegacy(stateDir: string, value: unknown): Promise<string> {
const sourcePath = path.join(stateDir, "restart-sentinel.json");
await fsp.writeFile(sourcePath, `${JSON.stringify(value)}\n`, "utf8");
return sourcePath;
}
async function migrate(params: {
env: NodeJS.ProcessEnv;
stateDir: string;
beforeVerify?: () => void;
removeSource?: (sourcePath: string) => Promise<void> | void;
}) {
return await migrateLegacyRestartSentinel({
detected: detectLegacyRestartSentinel({ stateDir: params.stateDir }),
...params,
});
}
function database(env: NodeJS.ProcessEnv) {
return openOpenClawStateDatabase({ env }).db;
}
function receipt(env: NodeJS.ProcessEnv) {
const db = database(env);
return executeSqliteQueryTakeFirstSync(
db,
getNodeSqliteKysely<MigrationDatabase>(db)
.selectFrom("migration_sources")
.selectAll()
.where("migration_kind", "=", "legacy-restart-sentinel-json"),
);
}
it("detects both the retired source and an interrupted fixed claim", async () => {
const { stateDir } = useStateDir();
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() });
expect(detectLegacyRestartSentinel({ stateDir }).hasLegacy).toBe(true);
await fsp.rename(sourcePath, `${sourcePath}.doctor-importing`);
expect(detectLegacyRestartSentinel({ stateDir }).hasLegacy).toBe(true);
});
it("imports and verifies the complete legacy payload before removing the source", async () => {
const { env, stateDir } = useStateDir();
const expected = payload();
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: expected });
const result = await migrate({ env, stateDir });
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
"Imported the legacy restart sentinel into shared SQLite state.",
]);
await expect(readRestartSentinel(env)).resolves.toMatchObject({
version: 1,
payload: expected,
});
expect(fs.existsSync(sourcePath)).toBe(false);
expect(receipt(env)).toMatchObject({
removed_source: 1,
source_record_count: 1,
status: "completed",
target_table: "gateway_restart_sentinel",
});
});
it("canonicalizes legacy null fields and an empty delivery context before verification", async () => {
const { env, stateDir } = useStateDir();
const sourcePath = await writeLegacy(stateDir, {
version: 1,
payload: {
kind: "restart",
status: "ok",
ts: 123,
deliveryContext: {},
message: null,
continuation: null,
doctorHint: null,
stats: null,
},
});
const result = await migrate({ env, stateDir });
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
"Imported the legacy restart sentinel into shared SQLite state.",
]);
const migrated = await readRestartSentinel(env);
expect(migrated?.payload).toEqual({ kind: "restart", status: "ok", ts: 123 });
expect(fs.existsSync(sourcePath)).toBe(false);
expect(receipt(env)).toMatchObject({
removed_source: 1,
source_record_count: 1,
status: "completed",
});
});
it("preserves a valid canonical row when legacy JSON conflicts", async () => {
const { env, stateDir } = useStateDir();
const canonical = payload(999);
await writeRestartSentinel(canonical, env);
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(1) });
const result = await migrate({ env, stateDir });
expect(result.changes).toEqual([
"Preserved the canonical SQLite restart sentinel and discarded conflicting legacy JSON.",
]);
await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: canonical });
expect(fs.existsSync(sourcePath)).toBe(false);
});
it("repairs an invalid canonical row from a validated legacy envelope", async () => {
const { env, stateDir } = useStateDir();
const db = database(env);
executeSqliteQuerySync(
db,
getNodeSqliteKysely<MigrationDatabase>(db).insertInto("gateway_restart_sentinel").values({
sentinel_key: "current",
version: 99,
kind: "update",
status: "ok",
ts: 1,
session_key: null,
thread_id: null,
delivery_channel: null,
delivery_to: null,
delivery_account_id: null,
message: null,
continuation_json: null,
doctor_hint: null,
stats_json: null,
payload_json: "{}",
updated_at_ms: 1,
}),
);
const expected = payload(456);
await writeLegacy(stateDir, { version: 1, payload: expected });
const result = await migrate({ env, stateDir });
expect(result.changes).toEqual([
"Replaced an invalid SQLite restart sentinel with validated legacy state.",
]);
await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: expected });
});
it("records and removes malformed transient state without disclosing its contents", async () => {
const { env, stateDir } = useStateDir();
const sourcePath = await writeLegacy(stateDir, {
version: 1,
payload: { ...payload(), ts: "invalid", message: "secret-marker" },
});
const result = await migrate({ env, stateDir });
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
"Discarded malformed retired restart sentinel JSON without importing it.",
]);
expect(JSON.stringify(result)).not.toContain("secret-marker");
expect(receipt(env)?.report_json).not.toContain("secret-marker");
await expect(readRestartSentinel(env)).resolves.toBeNull();
expect(fs.existsSync(sourcePath)).toBe(false);
});
it("treats a completed receipt as authoritative if the retired file reappears", async () => {
const { env, stateDir } = useStateDir();
await writeLegacy(stateDir, { version: 1, payload: payload(1) });
await migrate({ env, stateDir });
await clearRestartSentinel(env);
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(2) });
const result = await migrate({ env, stateDir });
expect(result.changes).toEqual([
"Discarded recreated retired restart sentinel JSON using its migration receipt.",
]);
await expect(readRestartSentinel(env)).resolves.toBeNull();
expect(fs.existsSync(sourcePath)).toBe(false);
});
it("recovers an interrupted claim and finishes the same migration owner", async () => {
const { env, stateDir } = useStateDir();
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() });
await fsp.rename(sourcePath, `${sourcePath}.doctor-importing`);
const result = await migrate({ env, stateDir });
expect(result.warnings).toEqual([]);
await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: payload() });
expect(fs.existsSync(sourcePath)).toBe(false);
expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false);
});
it("preserves changed source bytes and records no receipt", async () => {
const { env, stateDir } = useStateDir();
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(1) });
const result = await migrate({
env,
stateDir,
beforeVerify: () => {
fs.writeFileSync(sourcePath, JSON.stringify({ version: 1, payload: payload(2) }));
},
});
expect(result.warnings).toHaveLength(1);
expect(result.warnings[0]).toContain("changed after migration loaded it");
expect(fs.existsSync(sourcePath)).toBe(true);
expect(receipt(env)).toBeUndefined();
});
it("retains a claimed source after cleanup failure and converges on retry", async () => {
const { env, stateDir } = useStateDir();
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() });
const first = await migrate({
env,
stateDir,
removeSource: () => {
throw new Error("forced cleanup failure");
},
});
expect(first.warnings).toHaveLength(1);
expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(true);
expect(receipt(env)).toMatchObject({ removed_source: 0 });
const second = await migrate({ env, stateDir });
expect(second.warnings).toEqual([]);
expect(second.changes).toEqual([
"Discarded recreated retired restart sentinel JSON using its migration receipt.",
]);
expect(fs.existsSync(sourcePath)).toBe(false);
expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false);
expect(receipt(env)).toMatchObject({ removed_source: 1 });
});
it("requires exclusive state ownership before claiming the retired file", async () => {
const { env, stateDir } = useStateDir();
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() });
const gatewayLock = await acquireGatewayLock({
allowInTests: true,
env,
pollIntervalMs: 10,
port: 18_791,
timeoutMs: 100,
});
if (!gatewayLock) {
throw new Error("expected test Gateway lock");
}
let result: Awaited<ReturnType<typeof migrateLegacyRestartSentinel>>;
try {
result = await migrate({ env, stateDir });
} finally {
await gatewayLock.release();
}
expect(result.warnings[0]).toContain("Gateway or another SQLite maintenance command");
expect(fs.existsSync(sourcePath)).toBe(true);
expect(receipt(env)).toBeUndefined();
});
it("rejects symlinks, hardlinks, and oversized sources without deleting them", async () => {
const cases = ["symlink", "hardlink", "oversized"] as const;
for (const sourceKind of cases) {
closeOpenClawStateDatabaseForTest();
const { env, stateDir } = useStateDir();
const sourcePath = path.join(stateDir, "restart-sentinel.json");
if (sourceKind === "oversized") {
await fsp.writeFile(sourcePath, Buffer.alloc(4 * 1024 * 1024 + 1));
} else {
const targetPath = path.join(stateDir, `${sourceKind}-target.json`);
await fsp.writeFile(targetPath, JSON.stringify({ version: 1, payload: payload() }));
if (sourceKind === "symlink") {
await fsp.symlink(targetPath, sourcePath);
} else {
await fsp.link(targetPath, sourcePath);
}
}
const result = await migrate({ env, stateDir });
expect(result.warnings).toHaveLength(1);
expect(fs.existsSync(sourcePath)).toBe(true);
expect(receipt(env)).toBeUndefined();
}
});
});
@@ -0,0 +1,536 @@
// Startup/Doctor migration for the retired restart-sentinel JSON file.
import { createHash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
import { root, type Root } from "@openclaw/fs-safe";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
openOpenClawStateDatabase,
runOpenClawStateWriteTransaction,
} from "../state/openclaw-state-db.js";
import { formatErrorMessage } from "./errors.js";
import { acquireGatewayLock, GatewayLockError } from "./gateway-lock.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
import {
parseRestartSentinelEnvelope,
readRestartSentinelRowSync,
writeRestartSentinelRowSync,
type RestartSentinelEnvelope,
} from "./restart-sentinel-store.js";
import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js";
import type { MigrationMessages } from "./state-migrations.types.js";
const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json";
const DOCTOR_CLAIM_SUFFIX = ".doctor-importing";
const MAX_LEGACY_RESTART_SENTINEL_BYTES = 4 * 1024 * 1024;
const MIGRATION_KIND = "legacy-restart-sentinel-json";
const MIGRATION_LOCK_TIMEOUT_MS = 250;
const MIGRATION_LOCK_POLL_INTERVAL_MS = 25;
const utf8Decoder = new TextDecoder("utf-8", { fatal: true });
type RestartSentinelMigrationDatabase = Pick<
OpenClawStateKyselyDatabase,
"gateway_restart_sentinel" | "migration_runs" | "migration_sources"
>;
type LegacySourceSnapshot = {
buffer: Buffer;
dev: number;
ino: number;
mtimeMs: number;
sha256: string;
size: number;
};
type MigrationDecision =
| "canonical-preserved"
| "invalid-canonical-repaired"
| "legacy-imported"
| "malformed-legacy-discarded"
| "receipt-authoritative";
function legacyPathMayExist(filePath: string): boolean {
try {
fs.lstatSync(filePath);
return true;
} catch (error) {
return (error as NodeJS.ErrnoException).code !== "ENOENT";
}
}
/** Detect the exact retired file for startup preflight and explicit Doctor alike. */
export function detectLegacyRestartSentinel(params: {
stateDir: string;
}): LegacyRestartSentinelDetection {
const sourcePath = path.join(params.stateDir, LEGACY_RESTART_SENTINEL_FILENAME);
return {
sourcePath,
hasLegacy:
legacyPathMayExist(sourcePath) || legacyPathMayExist(`${sourcePath}${DOCTOR_CLAIM_SUFFIX}`),
};
}
function relativeLegacyPath(stateDir: string, filePath: string): string {
const relativePath = path.relative(path.resolve(stateDir), path.resolve(filePath));
if (
!relativePath ||
relativePath === ".." ||
relativePath.startsWith(`..${path.sep}`) ||
path.isAbsolute(relativePath)
) {
throw new Error("legacy restart sentinel path is outside the state directory");
}
return relativePath;
}
async function readLegacySourceSnapshot(
stateRoot: Root,
stateDir: string,
sourcePath: string,
): Promise<LegacySourceSnapshot> {
const opened = await stateRoot.read(relativeLegacyPath(stateDir, sourcePath), {
hardlinks: "reject",
maxBytes: MAX_LEGACY_RESTART_SENTINEL_BYTES,
symlinks: "reject",
});
if (!opened.stat.isFile() || opened.stat.size !== opened.buffer.byteLength) {
throw new Error("legacy restart sentinel is not a stable regular file");
}
return {
buffer: opened.buffer,
dev: opened.stat.dev,
ino: opened.stat.ino,
mtimeMs: opened.stat.mtimeMs,
sha256: createHash("sha256").update(opened.buffer).digest("hex"),
size: opened.stat.size,
};
}
function snapshotsMatch(left: LegacySourceSnapshot, right: LegacySourceSnapshot): boolean {
return (
left.dev === right.dev &&
left.ino === right.ino &&
left.mtimeMs === right.mtimeMs &&
left.sha256 === right.sha256 &&
left.size === right.size
);
}
function parseLegacyEnvelope(snapshot: LegacySourceSnapshot): RestartSentinelEnvelope | null {
try {
return parseRestartSentinelEnvelope(JSON.parse(utf8Decoder.decode(snapshot.buffer)));
} catch {
return null;
}
}
function receiptSourceKey(sourcePath: string): string {
return `restart-sentinel-json:${createHash("sha256").update(path.resolve(sourcePath)).digest("hex")}`;
}
function hasMigrationReceipt(sourcePath: string, env: NodeJS.ProcessEnv): boolean {
const { db } = openOpenClawStateDatabase({ env });
return Boolean(
executeSqliteQueryTakeFirstSync(
db,
getNodeSqliteKysely<RestartSentinelMigrationDatabase>(db)
.selectFrom("migration_sources")
.select("source_key")
.where("source_key", "=", receiptSourceKey(sourcePath)),
),
);
}
function decideAndRecordMigration(params: {
env: NodeJS.ProcessEnv;
sourcePath: string;
snapshot: LegacySourceSnapshot;
envelope: RestartSentinelEnvelope | null;
}): { decision: MigrationDecision; sourceKey: string } {
const sourceKey = receiptSourceKey(params.sourcePath);
const runId = `${sourceKey}:${params.snapshot.sha256.slice(0, 16)}`;
const now = Date.now();
return runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<RestartSentinelMigrationDatabase>(db);
const receipt = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("migration_sources")
.select("source_key")
.where("source_key", "=", sourceKey),
);
const before = readRestartSentinelRowSync(db);
let decision: MigrationDecision;
if (receipt) {
decision = "receipt-authoritative";
} else if (!params.envelope) {
decision = "malformed-legacy-discarded";
} else if (before.kind === "valid") {
decision = "canonical-preserved";
} else {
const written = writeRestartSentinelRowSync(db, params.envelope.payload);
const verified = readRestartSentinelRowSync(db);
if (
verified.kind !== "valid" ||
verified.sentinel.revision !== written.revision ||
!isDeepStrictEqual(verified.sentinel.payload, params.envelope.payload)
) {
throw new Error("SQLite verification failed for the restart sentinel migration");
}
decision = before.kind === "invalid" ? "invalid-canonical-repaired" : "legacy-imported";
}
const reportJson = JSON.stringify({
source: MIGRATION_KIND,
target: "gateway_restart_sentinel",
decision,
sourceSha256: params.snapshot.sha256,
sourceValid: params.envelope !== null,
importedRecordCount:
decision === "legacy-imported" || decision === "invalid-canonical-repaired" ? 1 : 0,
preservedSqliteRecordCount: decision === "canonical-preserved" ? 1 : 0,
});
executeSqliteQuerySync(
db,
stateDb
.insertInto("migration_runs")
.values({
id: runId,
started_at: now,
finished_at: now,
status: "completed",
report_json: reportJson,
})
.onConflict((conflict) =>
conflict.column("id").doUpdateSet({
finished_at: now,
status: "completed",
report_json: reportJson,
}),
),
);
executeSqliteQuerySync(
db,
stateDb
.insertInto("migration_sources")
.values({
source_key: sourceKey,
migration_kind: MIGRATION_KIND,
source_path: params.sourcePath,
target_table: "gateway_restart_sentinel",
source_sha256: params.snapshot.sha256,
source_size_bytes: params.snapshot.size,
source_record_count: params.envelope ? 1 : 0,
last_run_id: runId,
status: "completed",
imported_at: now,
removed_source: 0,
report_json: reportJson,
})
.onConflict((conflict) =>
conflict.column("source_key").doUpdateSet({
source_sha256: params.snapshot.sha256,
source_size_bytes: params.snapshot.size,
source_record_count: params.envelope ? 1 : 0,
last_run_id: runId,
status: "completed",
imported_at: now,
removed_source: 0,
report_json: reportJson,
}),
),
);
return { decision, sourceKey };
},
{ env: params.env },
);
}
function markSourceRemoved(sourceKey: string, env: NodeJS.ProcessEnv): void {
runOpenClawStateWriteTransaction(
({ db }) => {
executeSqliteQuerySync(
db,
getNodeSqliteKysely<RestartSentinelMigrationDatabase>(db)
.updateTable("migration_sources")
.set({ removed_source: 1 })
.where("source_key", "=", sourceKey),
);
},
{ env },
);
}
async function restoreClaim(params: {
stateRoot: Root;
stateDir: string;
sourcePath: string;
}): Promise<string | null> {
const claimPath = `${params.sourcePath}${DOCTOR_CLAIM_SUFFIX}`;
try {
if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, claimPath)))) {
return null;
}
if (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.sourcePath))) {
return `source path already exists: ${params.sourcePath}`;
}
await params.stateRoot.move(
relativeLegacyPath(params.stateDir, claimPath),
relativeLegacyPath(params.stateDir, params.sourcePath),
);
return null;
} catch (error) {
return String(error);
}
}
async function recoverInterruptedClaim(params: {
stateRoot: Root;
stateDir: string;
sourcePath: string;
env: NodeJS.ProcessEnv;
}): Promise<void> {
const claimPath = `${params.sourcePath}${DOCTOR_CLAIM_SUFFIX}`;
const claimRelativePath = relativeLegacyPath(params.stateDir, claimPath);
if (!(await params.stateRoot.exists(claimRelativePath))) {
return;
}
if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.sourcePath)))) {
await params.stateRoot.move(
claimRelativePath,
relativeLegacyPath(params.stateDir, params.sourcePath),
);
return;
}
// Both paths can only be retired safely when the claimed bytes already have
// an authoritative decision; otherwise preserve both for operator recovery.
if (!hasMigrationReceipt(params.sourcePath, params.env)) {
throw new Error("legacy restart sentinel source and interrupted claim both exist");
}
await readLegacySourceSnapshot(params.stateRoot, params.stateDir, claimPath);
await params.stateRoot.remove(claimRelativePath);
}
function decisionChange(decision: MigrationDecision): string {
switch (decision) {
case "legacy-imported":
return "Imported the legacy restart sentinel into shared SQLite state.";
case "invalid-canonical-repaired":
return "Replaced an invalid SQLite restart sentinel with validated legacy state.";
case "canonical-preserved":
return "Preserved the canonical SQLite restart sentinel and discarded conflicting legacy JSON.";
case "malformed-legacy-discarded":
return "Discarded malformed retired restart sentinel JSON without importing it.";
case "receipt-authoritative":
return "Discarded recreated retired restart sentinel JSON using its migration receipt.";
}
const unreachable: never = decision;
return unreachable;
}
async function migrateWithExclusiveStateOwnership(params: {
detected: LegacyRestartSentinelDetection;
stateRoot: Root;
stateDir: string;
env: NodeJS.ProcessEnv;
beforeClaim?: () => void;
beforeVerify?: () => void;
removeSource?: (sourcePath: string) => Promise<void> | void;
}): Promise<MigrationMessages> {
const changes: string[] = [];
const warnings: string[] = [];
const notices: string[] = [];
const sourcePath = params.detected.sourcePath;
try {
await recoverInterruptedClaim({
stateRoot: params.stateRoot,
stateDir: params.stateDir,
sourcePath,
env: params.env,
});
} catch (error) {
return {
changes,
warnings: [`Failed recovering a legacy restart sentinel Doctor claim: ${String(error)}`],
};
}
if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath)))) {
return { changes, warnings };
}
let snapshot: LegacySourceSnapshot;
try {
snapshot = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, sourcePath);
} catch (error) {
return {
changes,
warnings: [`Failed reading the legacy restart sentinel: ${String(error)}`],
};
}
const envelope = parseLegacyEnvelope(snapshot);
const claimPath = `${sourcePath}${DOCTOR_CLAIM_SUFFIX}`;
try {
params.beforeVerify?.();
const current = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, sourcePath);
if (!snapshotsMatch(current, snapshot)) {
throw new Error("legacy restart sentinel changed after migration loaded it");
}
params.beforeClaim?.();
await params.stateRoot.move(
relativeLegacyPath(params.stateDir, sourcePath),
relativeLegacyPath(params.stateDir, claimPath),
);
const claimed = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, claimPath);
if (!snapshotsMatch(claimed, snapshot)) {
throw new Error("legacy restart sentinel changed before migration could claim it");
}
} catch (error) {
const restoreError = await restoreClaim({
stateRoot: params.stateRoot,
stateDir: params.stateDir,
sourcePath,
});
return {
changes,
warnings: [
`Failed claiming the legacy restart sentinel: ${String(error)}${restoreError ? `; restore failure: ${restoreError}` : ""}`,
],
};
}
let result: ReturnType<typeof decideAndRecordMigration>;
try {
result = decideAndRecordMigration({
env: params.env,
sourcePath,
snapshot,
envelope,
});
} catch (error) {
const restoreError = await restoreClaim({
stateRoot: params.stateRoot,
stateDir: params.stateDir,
sourcePath,
});
return {
changes,
warnings: [
`Failed migrating the legacy restart sentinel: ${String(error)}${restoreError ? `; restore failure: ${restoreError}` : ""}`,
],
};
}
try {
if (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath))) {
throw new Error("legacy restart sentinel reappeared during migration cleanup");
}
if (params.removeSource) {
await params.removeSource(claimPath);
} else {
await params.stateRoot.remove(relativeLegacyPath(params.stateDir, claimPath));
}
if (
(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath))) ||
(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, claimPath)))
) {
throw new Error("legacy restart sentinel remains after migration cleanup");
}
} catch (error) {
warnings.push(`Legacy restart sentinel cleanup failed: ${String(error)}`);
return { changes, warnings };
}
try {
markSourceRemoved(result.sourceKey, params.env);
} catch (error) {
warnings.push(
`Legacy restart sentinel was removed, but its receipt could not be finalized: ${String(error)}`,
);
}
changes.push(decisionChange(result.decision));
notices.push("Removed retired restart-sentinel.json after recording its migration decision.");
return { changes, warnings, notices };
}
/** Import or retire the old file under exclusive state ownership. */
export async function migrateLegacyRestartSentinel(params: {
detected?: LegacyRestartSentinelDetection;
stateDir: string;
env?: NodeJS.ProcessEnv;
beforeClaim?: () => void;
beforeVerify?: () => void;
removeSource?: (sourcePath: string) => Promise<void> | void;
}): Promise<MigrationMessages> {
const detected = params.detected;
if (!detected?.hasLegacy) {
return { changes: [], warnings: [] };
}
const env = { ...(params.env ?? process.env), OPENCLAW_STATE_DIR: params.stateDir };
let lock: Awaited<ReturnType<typeof acquireGatewayLock>>;
try {
lock = await acquireGatewayLock({
allowInTests: true,
env,
pollIntervalMs: MIGRATION_LOCK_POLL_INTERVAL_MS,
role: "sqlite-maintenance",
timeoutMs: MIGRATION_LOCK_TIMEOUT_MS,
});
} catch (error) {
const detail =
error instanceof GatewayLockError
? "the Gateway or another SQLite maintenance command owns this state directory"
: String(error);
return {
changes: [],
warnings: [
`Failed migrating the legacy restart sentinel: ${detail}. Stop the Gateway, then run \`openclaw doctor --fix\` again.`,
],
};
}
if (!lock) {
return {
changes: [],
warnings: [
"Failed migrating the legacy restart sentinel: exclusive state ownership unavailable.",
],
};
}
let result: MigrationMessages = { changes: [], warnings: [] };
let releaseError: unknown;
try {
try {
const stateRoot = await root(params.stateDir, {
hardlinks: "reject",
maxBytes: MAX_LEGACY_RESTART_SENTINEL_BYTES,
symlinks: "reject",
});
result = await migrateWithExclusiveStateOwnership({
...params,
detected,
env,
stateRoot,
});
} catch (error) {
result.warnings.push(`Failed reading the legacy restart sentinel: ${String(error)}`);
}
} finally {
try {
await lock.release();
} catch (error) {
releaseError = error;
}
}
if (releaseError) {
result.warnings.push(
`Restart sentinel migration lock release failed: ${formatErrorMessage(releaseError)}`,
);
}
return result;
}
@@ -0,0 +1,4 @@
export type LegacyRestartSentinelDetection = {
sourcePath: string;
hasLegacy: boolean;
};
+19 -1
View File
@@ -28,6 +28,7 @@ import {
listWebPushSubscriptions,
readPersistedVapidKeyPair,
} from "./push-web-store.js";
import { readRestartSentinel } from "./restart-sentinel.js";
import {
autoMigrateLegacyState,
autoMigrateLegacyPluginDoctorState,
@@ -1729,7 +1730,7 @@ describe("state migrations", () => {
await expect(fs.readFile(`${routingPath}.migrated`, "utf8")).resolves.toContain("robot wake");
});
it("auto-migrates standalone legacy voice wake JSON settings", async () => {
it("auto-migrates standalone legacy JSON settings", async () => {
const root = await createTempDir();
const stateDir = path.join(root, ".openclaw");
const env = createEnv(stateDir);
@@ -1741,14 +1742,31 @@ describe("state migrations", () => {
JSON.stringify({ triggers: ["wake"] }),
"utf8",
);
const expectedSentinel = {
kind: "update" as const,
status: "ok" as const,
ts: 321,
message: "Update completed",
};
const restartSentinelPath = path.join(stateDir, "restart-sentinel.json");
await fs.writeFile(
restartSentinelPath,
`${JSON.stringify({ version: 1, payload: expectedSentinel })}\n`,
"utf8",
);
const result = await autoMigrateLegacyState({ cfg, env, homedir: () => root });
expect(result.skipped).toBe(false);
expect(result.migrated).toBe(true);
expect(result.warnings).toStrictEqual([]);
expect(result.changes).toContain(
"Imported the legacy restart sentinel into shared SQLite state.",
);
await expect(loadVoiceWakeConfig(stateDir)).resolves.toMatchObject({ triggers: ["wake"] });
await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: expectedSentinel });
await expectMissingPath(path.join(settingsDir, "voicewake.json"));
await expectMissingPath(restartSentinelPath);
});
it("runs plugin doctor migrations after repairing shared state schema", async () => {
+2
View File
@@ -3,6 +3,7 @@ import type { SessionScope } from "../config/sessions/types.js";
import type { PluginDoctorStateMigration } from "../plugins/doctor-contract-registry.js";
import type { LegacyChannelPairingStateDetection } from "./state-migrations.channel-pairing.js";
import type { LegacyMcpOAuthDetection } from "./state-migrations.mcp-oauth.types.js";
import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js";
import type { LegacyWorkspaceStateDetection } from "./state-migrations.workspace-setup.types.js";
export type LegacyRescuePendingDetection = {
@@ -115,6 +116,7 @@ export type LegacyStateDetection = {
hasLegacy: boolean;
};
mcpOauth: LegacyMcpOAuthDetection;
restartSentinel?: LegacyRestartSentinelDetection;
workspace: LegacyWorkspaceStateDetection;
webPush: {
subscriptionsPath: string;
@@ -113,10 +113,14 @@ function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): voi
doctorHint?: unknown;
stats?: unknown;
};
const revision =
typeof (sentinel as { revision?: unknown }).revision === "number"
? (sentinel as { revision: number }).revision
: Date.now();
executeSqliteQuerySync(
db,
stateDb.insertInto("gateway_restart_sentinel").values({
sentinel_key: "current",
sentinel_key: record.kind === "revision-floor" ? "revision-floor" : "current",
version: 1,
kind: typeof record.kind === "string" ? record.kind : "update",
status: typeof record.status === "string" ? record.status : "skipped",
@@ -136,22 +140,34 @@ function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): voi
doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null,
stats_json: record.stats ? JSON.stringify(record.stats) : null,
payload_json: JSON.stringify(payload),
updated_at_ms: Date.now(),
updated_at_ms: revision,
}),
);
}
function readRestartSentinelPayload(env: NodeJS.ProcessEnv): unknown {
function replaceRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "current"),
);
writeRestartSentinelRow(env, sentinel);
}
function readRestartSentinelPayload(env: NodeJS.ProcessEnv, key = "current"): unknown {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
const row = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select(["version", "payload_json"])
.where("sentinel_key", "=", "current"),
.select(["version", "payload_json", "updated_at_ms"])
.where("sentinel_key", "=", key),
);
return row ? { version: row.version, payload: JSON.parse(row.payload_json) } : null;
return row
? { version: row.version, payload: JSON.parse(row.payload_json), revision: row.updated_at_ms }
: null;
}
async function runHelperWithExistingSentinel(params: {
@@ -159,6 +175,8 @@ async function runHelperWithExistingSentinel(params: {
metaHandoffId?: string;
prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise<void> | void;
sentinel?: unknown;
parentExitTimeoutMs?: number;
whileHelperRunning?: (env: NodeJS.ProcessEnv) => Promise<void> | void;
}) {
const { execFile } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
@@ -206,7 +224,7 @@ async function runHelperWithExistingSentinel(params: {
{
...helperParams,
parentPid: process.pid,
parentExitTimeoutMs: 1,
parentExitTimeoutMs: params.parentExitTimeoutMs ?? 1,
stateDatabasePath: resolveOpenClawStateSqlitePath(env),
logPath: path.join(tmpDir, "handoff.log"),
sensitivePaths: [],
@@ -216,7 +234,7 @@ async function runHelperWithExistingSentinel(params: {
)}\n`,
);
const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
const resultPromise = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
(resolve) => {
execFile(process.execPath, [helperScriptPath, helperParamsPath], (err) => {
const childError = err as (NodeJS.ErrnoException & { signal?: NodeJS.Signals }) | null;
@@ -227,6 +245,8 @@ async function runHelperWithExistingSentinel(params: {
});
},
);
await params.whileHelperRunning?.(env);
const result = await resultPromise;
return { result, env };
}
@@ -868,7 +888,7 @@ describe("managed service update handoff", () => {
});
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toEqual(unrelatedSentinel);
expect(readRestartSentinelPayload(env)).toMatchObject(unrelatedSentinel);
});
it("does not overwrite a newer pending update handoff sentinel", async () => {
@@ -894,7 +914,74 @@ describe("managed service update handoff", () => {
});
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toEqual(newerSentinel);
expect(readRestartSentinelPayload(env)).toMatchObject(newerSentinel);
});
it("preserves a newer sentinel written while the detached helper is active", async () => {
const oldSentinel = {
version: 1,
revision: 100,
payload: {
kind: "update",
status: "skipped",
ts: 100,
stats: {
handoffId: "old-handoff",
reason: "managed-service-handoff-started",
},
},
};
const newerSentinel = {
version: 1,
revision: 200,
payload: {
kind: "restart",
status: "ok",
ts: 200,
},
};
const { env } = await runHelperWithExistingSentinel({
handoffId: "old-handoff",
metaHandoffId: "old-handoff",
sentinel: oldSentinel,
parentExitTimeoutMs: 200,
whileHelperRunning: async (stateEnv) => {
await new Promise<void>((resolve) => {
setTimeout(resolve, 50);
});
replaceRestartSentinelRow(stateEnv, newerSentinel);
},
});
expect(readRestartSentinelPayload(env)).toMatchObject({
payload: newerSentinel.payload,
revision: 200,
});
});
it("advances the durable revision floor even when it is ahead of the clock", async () => {
const futureRevision = Date.now() + 60_000;
const { env } = await runHelperWithExistingSentinel({
handoffId: "handoff-future-revision",
metaHandoffId: "handoff-future-revision",
sentinel: {
version: 1,
revision: futureRevision,
payload: {
kind: "revision-floor",
status: "skipped",
ts: 123,
stats: {
handoffId: "handoff-future-revision",
reason: "managed-service-handoff-started",
},
},
},
});
expect(readRestartSentinelPayload(env, "revision-floor")).toMatchObject({
revision: futureRevision + 1,
});
});
it("sweeps stale handoff temp directories while keeping fresh handoff logs", async () => {
+186 -75
View File
@@ -201,84 +201,140 @@ function hardenStateDatabaseFiles() {
}
}
function readRestartSentinelPayload() {
const db = openStateDatabase();
if (!db) {
function parseJsonColumn(value) {
if (typeof value !== "string" || !value) {
return null;
}
try {
const row = db
.prepare("SELECT version, payload_json FROM gateway_restart_sentinel WHERE sentinel_key = ?")
.get("current");
if (!row || row.version !== 1 || typeof row.payload_json !== "string") {
return null;
}
return JSON.parse(row.payload_json);
return JSON.parse(value);
} catch {
return null;
} finally {
hardenStateDatabaseFiles();
try {
db.close();
} catch {}
}
}
function writeRestartSentinelPayload(payload) {
const db = openStateDatabase();
if (!db) {
return;
function readRestartSentinelRecord(db) {
const row = db
.prepare(
[
"SELECT version, kind, status, ts, session_key, thread_id,",
"delivery_channel, delivery_to, delivery_account_id, message, continuation_json,",
"doctor_hint, stats_json, updated_at_ms",
"FROM gateway_restart_sentinel WHERE sentinel_key = ?",
].join(" "),
)
.get("current");
if (
!row ||
row.version !== 1 ||
typeof row.kind !== "string" ||
typeof row.status !== "string" ||
typeof row.ts !== "number" ||
typeof row.updated_at_ms !== "number"
) {
return null;
}
try {
const updatedAtMs = Date.now();
db.prepare(
const payload = {
kind: row.kind,
status: row.status,
ts: row.ts,
};
if (typeof row.session_key === "string") payload.sessionKey = row.session_key;
if (typeof row.thread_id === "string") payload.threadId = row.thread_id;
const deliveryContext = {};
if (typeof row.delivery_channel === "string") deliveryContext.channel = row.delivery_channel;
if (typeof row.delivery_to === "string") deliveryContext.to = row.delivery_to;
if (typeof row.delivery_account_id === "string") deliveryContext.accountId = row.delivery_account_id;
if (Object.keys(deliveryContext).length > 0) payload.deliveryContext = deliveryContext;
if (typeof row.message === "string") payload.message = row.message;
const continuation = parseJsonColumn(row.continuation_json);
if (continuation) payload.continuation = continuation;
if (typeof row.doctor_hint === "string") payload.doctorHint = row.doctor_hint;
const stats = parseJsonColumn(row.stats_json);
if (stats) payload.stats = stats;
return { revision: row.updated_at_ms, payload };
}
function readRestartSentinelRevisionFloor(db) {
const row = db
.prepare("SELECT updated_at_ms FROM gateway_restart_sentinel WHERE sentinel_key = ?")
.get("revision-floor");
if (!row) return null;
if (!Number.isSafeInteger(row.updated_at_ms)) {
throw new Error("restart sentinel revision floor is outside the safe integer range");
}
return row.updated_at_ms;
}
function advanceRestartSentinelRevisionFloor(db, revision) {
const payloadJson = JSON.stringify({ kind: "restart", status: "skipped", ts: revision });
db.prepare(
[
"INSERT INTO gateway_restart_sentinel (",
"sentinel_key, version, kind, status, ts, session_key, thread_id,",
"delivery_channel, delivery_to, delivery_account_id, message, continuation_json,",
"doctor_hint, stats_json, payload_json, updated_at_ms",
") VALUES ('revision-floor', 1, 'restart', 'skipped', ?, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ?, ?)",
"ON CONFLICT(sentinel_key) DO UPDATE SET",
"ts = excluded.ts, payload_json = excluded.payload_json, updated_at_ms = excluded.updated_at_ms",
].join(" "),
).run(revision, payloadJson, revision);
}
function writeRestartSentinelPayload(db, payload, currentRevision) {
const revisionFloor = readRestartSentinelRevisionFloor(db);
const updatedAtMs = Math.max(Date.now(), Math.max(currentRevision || 0, revisionFloor || 0) + 1);
if (!Number.isSafeInteger(updatedAtMs)) {
throw new Error("restart sentinel revision exhausted the safe integer range");
}
const values = [
payload.kind,
payload.status,
payload.ts,
payload.sessionKey || null,
payload.threadId || null,
payload.deliveryContext && typeof payload.deliveryContext.channel === "string"
? payload.deliveryContext.channel
: null,
payload.deliveryContext && typeof payload.deliveryContext.to === "string"
? payload.deliveryContext.to
: null,
payload.deliveryContext && typeof payload.deliveryContext.accountId === "string"
? payload.deliveryContext.accountId
: null,
payload.message || null,
payload.continuation ? JSON.stringify(payload.continuation) : null,
payload.doctorHint || null,
payload.stats ? JSON.stringify(payload.stats) : null,
JSON.stringify(payload),
updatedAtMs,
];
let changed;
if (currentRevision === null) {
changed = db.prepare(
[
"INSERT INTO gateway_restart_sentinel (",
"sentinel_key, version, kind, status, ts, session_key, thread_id,",
"delivery_channel, delivery_to, delivery_account_id, message, continuation_json,",
"doctor_hint, stats_json, payload_json, updated_at_ms",
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"ON CONFLICT(sentinel_key) DO UPDATE SET",
"version = excluded.version, kind = excluded.kind, status = excluded.status,",
"ts = excluded.ts, session_key = excluded.session_key, thread_id = excluded.thread_id,",
"delivery_channel = excluded.delivery_channel, delivery_to = excluded.delivery_to,",
"delivery_account_id = excluded.delivery_account_id, message = excluded.message,",
"continuation_json = excluded.continuation_json, doctor_hint = excluded.doctor_hint,",
"stats_json = excluded.stats_json, payload_json = excluded.payload_json,",
"updated_at_ms = excluded.updated_at_ms",
") VALUES ('current', 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
].join(" "),
).run(
"current",
1,
payload.kind,
payload.status,
payload.ts,
payload.sessionKey || null,
payload.threadId || null,
payload.deliveryContext && typeof payload.deliveryContext.channel === "string"
? payload.deliveryContext.channel
: null,
payload.deliveryContext && typeof payload.deliveryContext.to === "string"
? payload.deliveryContext.to
: null,
payload.deliveryContext && typeof payload.deliveryContext.accountId === "string"
? payload.deliveryContext.accountId
: null,
payload.message || null,
payload.continuation ? JSON.stringify(payload.continuation) : null,
payload.doctorHint || null,
payload.stats ? JSON.stringify(payload.stats) : null,
JSON.stringify(payload),
updatedAtMs,
);
} catch (err) {
appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err)));
} finally {
hardenStateDatabaseFiles();
try {
db.close();
} catch {}
).run(...values).changes === 1;
} else {
changed = db.prepare(
[
"UPDATE gateway_restart_sentinel SET",
"version = 1, kind = ?, status = ?, ts = ?, session_key = ?, thread_id = ?,",
"delivery_channel = ?, delivery_to = ?, delivery_account_id = ?, message = ?,",
"continuation_json = ?, doctor_hint = ?, stats_json = ?, payload_json = ?, updated_at_ms = ?",
"WHERE sentinel_key = 'current' AND updated_at_ms = ?",
].join(" "),
).run(...values, currentRevision).changes === 1;
}
if (changed) {
// This runs inside the same BEGIN IMMEDIATE section as the guarded current-row write.
advanceRestartSentinelRevisionFloor(db, updatedAtMs);
}
return changed;
}
function buildFallbackFailurePayload(reason) {
@@ -312,22 +368,77 @@ function buildFallbackFailurePayload(reason) {
}
function markUpdateSentinelFailureIfPending(reason) {
let payload = readRestartSentinelPayload();
if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) {
const snapshotDb = openStateDatabase();
if (!snapshotDb) return;
let snapshot;
try {
snapshot = readRestartSentinelRecord(snapshotDb);
} catch {
return;
} finally {
try {
snapshotDb.close();
} catch {}
}
const handoffId = typeof params.handoffId === "string" ? params.handoffId.trim() : "";
if (payload && handoffId && (!payload.stats || payload.stats.handoffId !== handoffId)) {
return;
const fallbackPayload = snapshot === null ? buildFallbackFailurePayload(reason) : null;
const db = openStateDatabase();
if (!db) return;
let transactionOpen = false;
try {
db.exec("BEGIN IMMEDIATE;");
transactionOpen = true;
const current = readRestartSentinelRecord(db);
if (
(snapshot === null && current !== null) ||
(snapshot !== null &&
(current === null || current.revision !== snapshot.revision))
) {
db.exec("COMMIT;");
transactionOpen = false;
return;
}
let payload = current && current.payload;
if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) {
db.exec("COMMIT;");
transactionOpen = false;
return;
}
const handoffId = typeof params.handoffId === "string" ? params.handoffId.trim() : "";
if (payload && handoffId && (!payload.stats || payload.stats.handoffId !== handoffId)) {
db.exec("COMMIT;");
transactionOpen = false;
return;
}
if (payload) {
payload = { ...payload, status: "error" };
delete payload.continuation;
payload.stats = { ...(payload.stats || {}), reason };
} else {
payload = fallbackPayload;
}
if (!payload) {
throw new Error("restart sentinel disappeared before guarded failure write");
}
if (!writeRestartSentinelPayload(db, payload, current ? current.revision : null)) {
throw new Error("restart sentinel changed before guarded failure write");
}
db.exec("COMMIT;");
transactionOpen = false;
} catch (err) {
if (transactionOpen) {
try {
db.exec("ROLLBACK;");
} catch {}
}
appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err)));
} finally {
hardenStateDatabaseFiles();
try {
db.close();
} catch {}
}
if (payload) {
payload = { ...payload, status: "error" };
delete payload.continuation;
payload.stats = { ...(payload.stats || {}), reason };
} else {
payload = buildFallbackFailurePayload(reason);
}
writeRestartSentinelPayload(payload);
}
function runServiceCommand(command, args) {
@@ -128,6 +128,84 @@ describe("check-database-first-legacy-stores", () => {
expect(violations).toEqual([{ kind: "legacy store filesystem write", line: 5 }]);
});
it("keeps legacy restart sentinel filesystem access in its sole migration owner", () => {
const runtimeViolations = collectDatabaseFirstLegacyStoreViolations(
`
import { readFile } from "node:fs/promises";
import path from "node:path";
const legacyFilename = "restart-sentinel.json";
`,
"src/infra/restart-sentinel.ts",
);
const migrationViolations = collectDatabaseFirstLegacyStoreViolations(
`
import { readFile } from "node:fs/promises";
import path from "node:path";
const legacyFilename = "restart-sentinel.json";
`,
"src/infra/state-migrations.restart-sentinel.ts",
);
expect(runtimeViolations).toEqual([
{ kind: "legacy restart sentinel filesystem import", line: 2 },
{ kind: "legacy restart sentinel filesystem import", line: 3 },
{ kind: "legacy restart sentinel reference", line: 4 },
]);
expect(migrationViolations).toEqual([]);
});
it("flags legacy restart sentinel references outside the migration owner", () => {
const violations = collectDatabaseFirstLegacyStoreViolations(
`
const legacyPath = path.join(stateDir, "restart-sentinel.json");
await readFile(legacyPath, "utf8");
`,
"src/commands/doctor/state-migrations.ts",
);
expect(violations).toEqual([{ kind: "legacy restart sentinel reference", line: 2 }]);
});
it("allows the CLI preflight to detect exact legacy restart sentinel inputs", () => {
const violations = collectDatabaseFirstLegacyStoreViolations(
`
[
path.join(stateDir, "restart-sentinel.json"),
path.join(stateDir, "restart-sentinel.json.doctor-importing"),
].some(fileOrDirExists);
`,
"src/cli/program/config-guard.ts",
);
expect(violations).toEqual([]);
});
it("flags direct legacy restart sentinel reads from the CLI preflight", () => {
const violations = collectDatabaseFirstLegacyStoreViolations(
`
await readFile(path.join(stateDir, "restart-sentinel.json"), "utf8");
await readFile(path.join(stateDir, "restart-sentinel.json.doctor-importing"), "utf8");
`,
"src/cli/program/config-guard.ts",
);
expect(violations).toEqual([
{ kind: "legacy restart sentinel reference", line: 2 },
{ kind: "legacy restart sentinel reference", line: 3 },
]);
});
it("flags nested restart sentinel paths disguised as CLI preflight detection", () => {
const violations = collectDatabaseFirstLegacyStoreViolations(
`
[path.join(stateDir, "archive/restart-sentinel.json")].some(fileOrDirExists);
`,
"src/cli/program/config-guard.ts",
);
expect(violations).toEqual([{ kind: "legacy restart sentinel reference", line: 2 }]);
});
it("flags retired Diffs viewer sidecar writes", () => {
const violations = collectDatabaseFirstLegacyStoreViolations(
`
@@ -1819,6 +1897,7 @@ describe("check-database-first-legacy-stores", () => {
);
expect(violations).toEqual([
{ kind: "legacy restart sentinel reference", line: 5 },
{ kind: "legacy store filesystem write", line: 5 },
{ kind: "legacy store filesystem write", line: 6 },
{ kind: "legacy store filesystem write", line: 7 },