mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
refactor: store gateway restart recovery in SQLite (#110014)
* refactor: move restart sentinel state to sqlite * fix: satisfy restart sentinel architecture gates * test: avoid cold startup migration timeout * fix: bound durable restart notice retries * test: type restart notice contention mock * fix: detect legacy restart sentinel before reads * fix: keep delivery attempt result internal * fix: narrow restart sentinel preflight guard
This commit is contained in:
committed by
GitHub
parent
b75b62fb1c
commit
5366b6de81
@@ -159,6 +159,14 @@ SQLite before the process exits. After boot the gateway posts the outcome back
|
||||
to the originating chat and dispatches a one-shot continuation turn so the
|
||||
agent picks up exactly where it left off, on the same channel and thread.
|
||||
|
||||
The sentinel's typed SQLite columns are authoritative for restart handling;
|
||||
its `payload_json` value is a replay/debug shadow only. Runtime reads, writes,
|
||||
and clears SQLite state without a file fallback. During the storage cutover, a
|
||||
bounded state migration runs at startup and through Doctor to preserve a
|
||||
validated `restart-sentinel.json` left by the older process after an update.
|
||||
The migration verifies the typed row and removes the source file before normal
|
||||
restart handling continues.
|
||||
|
||||
## Safety valves and observability
|
||||
|
||||
- **Crash-loop breaker:** 3 unclean boots within 5 minutes trip a breaker that
|
||||
|
||||
@@ -1256,8 +1256,12 @@ sessionId})`; create, branch, continue, list, and fork flows live in their
|
||||
- Gateway restart sentinel state now uses typed shared SQLite
|
||||
`gateway_restart_sentinel` rows instead of `restart-sentinel.json`; runtime
|
||||
reads sentinel kind, status, routing, message, continuation, and stats from
|
||||
typed columns. `payload_json` is only a replay/debug copy. Runtime code clears
|
||||
the SQLite row directly and no longer carries file cleanup plumbing.
|
||||
typed columns. Those columns are authoritative; `payload_json` is only a
|
||||
replay/debug shadow. Runtime read, write, and clear paths are SQLite-only.
|
||||
One bounded state-migration module runs during startup and Doctor to import a
|
||||
validated older post-update sentinel before normal restart recovery, verify
|
||||
the typed row, and remove the source file. No steady-state runtime module
|
||||
reads, writes, or cleans up the legacy file.
|
||||
- Gateway restart intent and supervisor handoff state now use typed shared
|
||||
SQLite `gateway_restart_intent` and `gateway_restart_handoff` rows instead of
|
||||
`gateway-restart-intent.json` and
|
||||
|
||||
@@ -84,6 +84,26 @@ const fsSafePackageModulePattern = /^@openclaw\/fs-safe(?:\/(?:root|store))?$/u;
|
||||
|
||||
const bridgeMarkerPattern = /\btranscriptLocator\b|sqlite-transcript:\/\//u;
|
||||
|
||||
// The restart handoff must survive its one cutover migration without leaving
|
||||
// filesystem fallback imports in the steady-state runtime owner.
|
||||
const legacyRestartSentinelMigrationPath = "src/infra/state-migrations.restart-sentinel.ts";
|
||||
const legacyRestartSentinelPreflightPath = "src/cli/program/config-guard.ts";
|
||||
const legacyRestartSentinelRuntimePath = "src/infra/restart-sentinel.ts";
|
||||
const legacyRestartSentinelPreflightFilenames = new Set([
|
||||
"restart-sentinel.json",
|
||||
"restart-sentinel.json.doctor-importing",
|
||||
]);
|
||||
const legacyRestartSentinelFilenamePattern =
|
||||
/(?:^|[/\\])restart-sentinel\.json(?:\.doctor-importing)?$/u;
|
||||
const legacyRestartSentinelRuntimeImportSpecifiers = new Set([
|
||||
"fs",
|
||||
"fs/promises",
|
||||
"node:fs",
|
||||
"node:fs/promises",
|
||||
"node:path",
|
||||
"path",
|
||||
]);
|
||||
|
||||
const legacyStorePatterns = [
|
||||
/\bsessions\.json\b/u,
|
||||
/\.trajectory\.jsonl\b/u,
|
||||
@@ -129,6 +149,7 @@ const allowedRuntimeMigrationPaths = [
|
||||
"src/infra/state-migrations.managed-outgoing-images.ts",
|
||||
"src/infra/state-migrations.apns.ts",
|
||||
"src/infra/state-migrations.mcp-oauth.ts",
|
||||
legacyRestartSentinelMigrationPath,
|
||||
"src/infra/state-migrations.workspace-setup.ts",
|
||||
"src/infra/state-migrations.web-push.ts",
|
||||
"src/infra/state-migrations.node-host.ts",
|
||||
@@ -327,6 +348,86 @@ function importSource(node) {
|
||||
return ts.isStringLiteral(moduleSpecifier) ? moduleSpecifier.text : "";
|
||||
}
|
||||
|
||||
function isLegacyRestartSentinelPreflightDetection(node, relativePath) {
|
||||
if (
|
||||
relativePath !== legacyRestartSentinelPreflightPath ||
|
||||
!legacyRestartSentinelPreflightFilenames.has(node.text)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const joinCall = node.parent;
|
||||
if (
|
||||
!ts.isCallExpression(joinCall) ||
|
||||
joinCall.arguments.length !== 2 ||
|
||||
joinCall.arguments[1] !== node ||
|
||||
!ts.isPropertyAccessExpression(joinCall.expression) ||
|
||||
!ts.isIdentifier(joinCall.expression.expression) ||
|
||||
joinCall.expression.expression.text !== "path" ||
|
||||
joinCall.expression.name.text !== "join" ||
|
||||
!ts.isIdentifier(joinCall.arguments[0]) ||
|
||||
joinCall.arguments[0].text !== "stateDir"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const paths = joinCall.parent;
|
||||
if (!ts.isArrayLiteralExpression(paths)) {
|
||||
return false;
|
||||
}
|
||||
const someAccess = paths.parent;
|
||||
if (
|
||||
!ts.isPropertyAccessExpression(someAccess) ||
|
||||
someAccess.expression !== paths ||
|
||||
someAccess.name.text !== "some"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const someCall = someAccess.parent;
|
||||
return (
|
||||
ts.isCallExpression(someCall) &&
|
||||
someCall.arguments.length === 1 &&
|
||||
ts.isIdentifier(someCall.arguments[0]) &&
|
||||
someCall.arguments[0].text === "fileOrDirExists"
|
||||
);
|
||||
}
|
||||
|
||||
function collectLegacyRestartSentinelBoundaryViolations(sourceFile, relativePath) {
|
||||
if (relativePath === legacyRestartSentinelMigrationPath) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const violations = [];
|
||||
const seen = new Set();
|
||||
function add(node, kind) {
|
||||
const line = toLine(sourceFile, node);
|
||||
const key = `${line}:${kind}`;
|
||||
if (seen.has(key)) {
|
||||
return;
|
||||
}
|
||||
seen.add(key);
|
||||
violations.push({ kind, line });
|
||||
}
|
||||
|
||||
function visit(node) {
|
||||
if (
|
||||
ts.isStringLiteralLike(node) &&
|
||||
legacyRestartSentinelFilenamePattern.test(node.text) &&
|
||||
!isLegacyRestartSentinelPreflightDetection(node, relativePath)
|
||||
) {
|
||||
add(node, "legacy restart sentinel reference");
|
||||
}
|
||||
if (
|
||||
relativePath === legacyRestartSentinelRuntimePath &&
|
||||
ts.isImportDeclaration(node) &&
|
||||
legacyRestartSentinelRuntimeImportSpecifiers.has(importSource(node))
|
||||
) {
|
||||
add(node, "legacy restart sentinel filesystem import");
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
}
|
||||
visit(sourceFile);
|
||||
return violations;
|
||||
}
|
||||
|
||||
function isHelperWriteModuleSource(source) {
|
||||
return (
|
||||
source === "openclaw/plugin-sdk/file-access-runtime" ||
|
||||
@@ -526,21 +627,28 @@ function legacyCandidateTexts(sourceFile, node) {
|
||||
*/
|
||||
export function collectDatabaseFirstLegacyStoreViolations(
|
||||
content,
|
||||
relativePath = "source.ts",
|
||||
inputRelativePath = "source.ts",
|
||||
scanOptions = {},
|
||||
) {
|
||||
const relativePath = inputRelativePath.replaceAll("\\", "/");
|
||||
const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true);
|
||||
const boundaryViolations = collectLegacyRestartSentinelBoundaryViolations(
|
||||
sourceFile,
|
||||
relativePath,
|
||||
);
|
||||
if (isAllowedLegacyOwnerPath(relativePath)) {
|
||||
return [];
|
||||
return boundaryViolations;
|
||||
}
|
||||
|
||||
const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true);
|
||||
const currentLegacyWriteAllowances =
|
||||
scanOptions.currentLegacyWriteAllowances ?? currentLegacyWriteViolationAllowances(relativePath);
|
||||
const createRequireBindings = collectCreateRequireBindings(sourceFile);
|
||||
const { fsModuleBindings, fsWriteAliases, fsSafeStoreFactoryAliases } =
|
||||
collectFsBindings(sourceFile);
|
||||
const violations = [];
|
||||
const seenViolations = new Set();
|
||||
const violations = [...boundaryViolations];
|
||||
const seenViolations = new Set(
|
||||
boundaryViolations.map((violation) => `${violation.line}:${violation.kind}`),
|
||||
);
|
||||
const fsModuleBindingScopes = [new Map([...fsModuleBindings].map((name) => [name, true]))];
|
||||
const fsModulePropertyScopes = [new Map()];
|
||||
const fsWriteAliasScopes = [fsWriteAliases];
|
||||
|
||||
@@ -207,6 +207,23 @@ describe("ensureConfigReady", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["restart-sentinel.json", "restart-sentinel.json.doctor-importing"])(
|
||||
"runs doctor flow when lightweight startup detection finds %s",
|
||||
async (relativePath) => {
|
||||
const root = useTempOpenClawHome();
|
||||
writeStateMarker(root, relativePath);
|
||||
|
||||
await runEnsureConfigReady(["status"]);
|
||||
|
||||
expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({
|
||||
migrateState: true,
|
||||
migrateLegacyConfig: false,
|
||||
invalidConfigNote: false,
|
||||
observe: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("runs doctor flow when lightweight startup detection finds a pending SQLite archive", async () => {
|
||||
const root = useTempOpenClawHome();
|
||||
writePendingTaskSidecarArchiveMarker(root);
|
||||
|
||||
@@ -124,6 +124,8 @@ function hasLegacyStateMigrationInputs(): boolean {
|
||||
path.join(stateDir, "agent"),
|
||||
path.join(stateDir, "agents"),
|
||||
path.join(stateDir, "plugins", "installs.json"),
|
||||
path.join(stateDir, "restart-sentinel.json"),
|
||||
path.join(stateDir, "restart-sentinel.json.doctor-importing"),
|
||||
path.join(stateDir, "sessions"),
|
||||
path.join(stateDir, "state", "openclaw.sqlite"),
|
||||
].some(fileOrDirExists) ||
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
// Exercises restart-notice retries against the real SQLite outbound queue.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { getDeliveryQueueEntryStatus } from "../infra/delivery-queue-sqlite.js";
|
||||
import { PlatformMessageNotDispatchedError } from "../infra/outbound/deliver-types.js";
|
||||
import { loadPendingDelivery } from "../infra/outbound/delivery-queue-storage.js";
|
||||
import { markDeliveryPlatformSendAttemptStarted } from "../infra/outbound/delivery-queue.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
sendDurableMessageBatch: vi.fn(),
|
||||
recoveryDeliver: vi.fn(),
|
||||
resolveOutboundChannelMessageAdapter: vi.fn(() => undefined),
|
||||
sleep: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
vi.mock("../channels/message/runtime.js", () => ({
|
||||
sendDurableMessageBatch: mocks.sendDurableMessageBatch,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/outbound/deliver.js", () => ({
|
||||
deliverOutboundPayloadsInternal: mocks.recoveryDeliver,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/outbound/channel-resolution.js", () => ({
|
||||
resolveOutboundChannelMessageAdapter: mocks.resolveOutboundChannelMessageAdapter,
|
||||
}));
|
||||
|
||||
vi.mock("../utils/sleep.js", () => ({ sleep: mocks.sleep }));
|
||||
|
||||
const { deliverRestartSentinelNotice, enqueueRestartSentinelNotice } =
|
||||
await import("./server-restart-sentinel-notice.js");
|
||||
|
||||
type DeliveryRequest = { deliveryQueueId?: string; deliveryQueueStateDir?: string };
|
||||
|
||||
describe("restart sentinel notice recovery", () => {
|
||||
let envSnapshot: ReturnType<typeof captureEnv> | undefined;
|
||||
let stateDir = "";
|
||||
const tempDirs = useAutoCleanupTempDirTracker((cleanup) => {
|
||||
afterEach(() => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
envSnapshot?.restore();
|
||||
envSnapshot = undefined;
|
||||
cleanup();
|
||||
});
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
stateDir = tempDirs.make("openclaw-restart-notice-");
|
||||
envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
mocks.sendDurableMessageBatch.mockReset();
|
||||
mocks.recoveryDeliver.mockReset();
|
||||
mocks.resolveOutboundChannelMessageAdapter.mockClear();
|
||||
mocks.sleep.mockClear();
|
||||
});
|
||||
|
||||
async function enqueueNotice(): Promise<string> {
|
||||
const queued = await enqueueRestartSentinelNotice({
|
||||
channel: "whatsapp",
|
||||
to: "+15550002",
|
||||
message: "restart complete",
|
||||
sessionKey: "agent:main:main",
|
||||
revision: 123,
|
||||
});
|
||||
return queued.id;
|
||||
}
|
||||
|
||||
async function deliverNotice(queueId: string): Promise<void> {
|
||||
await deliverRestartSentinelNotice({
|
||||
deps: {} as never,
|
||||
cfg: {},
|
||||
channel: "whatsapp",
|
||||
to: "+15550002",
|
||||
message: "restart complete",
|
||||
sessionKey: "agent:main:main",
|
||||
summary: "restart summary",
|
||||
queueId,
|
||||
});
|
||||
}
|
||||
|
||||
async function markAttempt(request: unknown): Promise<void> {
|
||||
const { deliveryQueueId, deliveryQueueStateDir } = request as DeliveryRequest;
|
||||
if (!deliveryQueueId) {
|
||||
throw new Error("expected durable delivery queue id");
|
||||
}
|
||||
await markDeliveryPlatformSendAttemptStarted(deliveryQueueId, deliveryQueueStateDir);
|
||||
}
|
||||
|
||||
function queueStatus(queueId: string): string | undefined {
|
||||
return getDeliveryQueueEntryStatus("outbound", queueId, stateDir);
|
||||
}
|
||||
|
||||
it("replays a retryable provider-not-dispatched failure after the startup scan", async () => {
|
||||
const queueId = await enqueueNotice();
|
||||
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
|
||||
await markAttempt(request);
|
||||
return {
|
||||
status: "failed",
|
||||
error: new PlatformMessageNotDispatchedError("connect failed before dispatch", {
|
||||
cause: new Error("connect failed"),
|
||||
}),
|
||||
};
|
||||
});
|
||||
mocks.recoveryDeliver.mockResolvedValueOnce([
|
||||
{ channel: "whatsapp", messageId: "recovered-1" },
|
||||
]);
|
||||
|
||||
await deliverNotice(queueId);
|
||||
|
||||
expect(mocks.recoveryDeliver).toHaveBeenCalledOnce();
|
||||
expect(await loadPendingDelivery(queueId)).toBeNull();
|
||||
expect(queueStatus(queueId)).toBe("completed");
|
||||
});
|
||||
|
||||
it("does not blindly resend an ambiguous platform attempt", async () => {
|
||||
const queueId = await enqueueNotice();
|
||||
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
|
||||
await markAttempt(request);
|
||||
return { status: "failed", error: new Error("platform outcome unknown") };
|
||||
});
|
||||
|
||||
await deliverNotice(queueId);
|
||||
|
||||
expect(mocks.recoveryDeliver).not.toHaveBeenCalled();
|
||||
expect(await loadPendingDelivery(queueId)).toBeNull();
|
||||
expect(queueStatus(queueId)).toBe("failed");
|
||||
});
|
||||
|
||||
it("dead-letters a permanent provider rejection without replay", async () => {
|
||||
const queueId = await enqueueNotice();
|
||||
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
|
||||
await markAttempt(request);
|
||||
return {
|
||||
status: "failed",
|
||||
error: new PlatformMessageNotDispatchedError("payload rejected", {
|
||||
cause: new Error("invalid payload"),
|
||||
retryable: false,
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
await deliverNotice(queueId);
|
||||
|
||||
expect(mocks.recoveryDeliver).not.toHaveBeenCalled();
|
||||
expect(await loadPendingDelivery(queueId)).toBeNull();
|
||||
expect(queueStatus(queueId)).toBe("failed");
|
||||
});
|
||||
|
||||
it("preserves the shipped 45-attempt budget before dead-lettering", async () => {
|
||||
const queueId = await enqueueNotice();
|
||||
const retryableFailure = () =>
|
||||
new PlatformMessageNotDispatchedError("transport unavailable before dispatch", {
|
||||
cause: new Error("transport unavailable"),
|
||||
});
|
||||
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
|
||||
await markAttempt(request);
|
||||
return { status: "failed", error: retryableFailure() };
|
||||
});
|
||||
mocks.recoveryDeliver.mockImplementation(async (request) => {
|
||||
await markAttempt(request);
|
||||
throw retryableFailure();
|
||||
});
|
||||
|
||||
await deliverNotice(queueId);
|
||||
|
||||
expect(mocks.sendDurableMessageBatch).toHaveBeenCalledOnce();
|
||||
expect(mocks.recoveryDeliver).toHaveBeenCalledTimes(44);
|
||||
expect(queueStatus(queueId)).toBe("failed");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,260 @@
|
||||
// Durable outbound notice ownership for restart-sentinel recovery.
|
||||
import { sendDurableMessageBatch } from "../channels/message/runtime.js";
|
||||
import type { CliDeps } from "../cli/deps.types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
findPlatformMessageRejectedError,
|
||||
isProvenDeliveryNotSentError,
|
||||
} from "../infra/delivery-recovery.shared.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { deliverOutboundPayloadsInternal } from "../infra/outbound/deliver.js";
|
||||
import {
|
||||
failPendingDelivery,
|
||||
loadPendingDelivery,
|
||||
reserveDeliveryAttempt,
|
||||
} from "../infra/outbound/delivery-queue-storage.js";
|
||||
import {
|
||||
ackDelivery,
|
||||
drainPendingDeliveries,
|
||||
enqueueDeliveryOnce,
|
||||
failDelivery,
|
||||
failDeliveryAfterPlatformSend,
|
||||
failDeliveryBeforePlatformSend,
|
||||
withActiveDeliveryClaim,
|
||||
} from "../infra/outbound/delivery-queue.js";
|
||||
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
|
||||
const log = createSubsystemLogger("gateway/restart-sentinel");
|
||||
const RESTART_NOTICE_RECOVERY_DELAY_MS = process.env.VITEST ? 1 : 1_000;
|
||||
const RESTART_NOTICE_MAX_ATTEMPTS = 45;
|
||||
const RESTART_NOTICE_RECOVERY_MAX_CYCLES = RESTART_NOTICE_MAX_ATTEMPTS + 1;
|
||||
|
||||
type RestartSentinelNoticeRoute = {
|
||||
channel: string;
|
||||
to: string;
|
||||
accountId?: string;
|
||||
replyToId?: string;
|
||||
threadId?: string;
|
||||
};
|
||||
|
||||
export async function enqueueRestartSentinelNotice(
|
||||
params: RestartSentinelNoticeRoute & {
|
||||
message: string;
|
||||
sessionKey: string;
|
||||
revision: number;
|
||||
},
|
||||
): Promise<{ id: string; created: boolean }> {
|
||||
return await enqueueDeliveryOnce(
|
||||
{
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
accountId: params.accountId,
|
||||
replyToId: params.replyToId,
|
||||
threadId: params.threadId,
|
||||
payloads: [{ text: params.message }],
|
||||
bestEffort: false,
|
||||
completionRetention: "permanent",
|
||||
maxRetries: RESTART_NOTICE_MAX_ATTEMPTS,
|
||||
},
|
||||
`restart-sentinel-notice:${params.sessionKey}:${params.revision}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForRecoveryDrain(): Promise<void> {
|
||||
await new Promise<void>((resolve) => {
|
||||
const timer = setTimeout(resolve, RESTART_NOTICE_RECOVERY_DELAY_MS);
|
||||
timer.unref?.();
|
||||
});
|
||||
}
|
||||
|
||||
async function drainFailedRestartSentinelNotice(params: {
|
||||
cfg: OpenClawConfig;
|
||||
queueId: string;
|
||||
sessionKey: string;
|
||||
summary: string;
|
||||
}): Promise<void> {
|
||||
for (let cycle = 1; cycle <= RESTART_NOTICE_RECOVERY_MAX_CYCLES; cycle += 1) {
|
||||
const beforeDrain = await loadPendingDelivery(params.queueId).catch((error: unknown) => {
|
||||
log.warn(`${params.summary}: restart notice recovery reload failed: ${String(error)}`, {
|
||||
queueId: params.queueId,
|
||||
sessionKey: params.sessionKey,
|
||||
cycle,
|
||||
});
|
||||
return undefined;
|
||||
});
|
||||
if (beforeDrain === null) {
|
||||
return;
|
||||
}
|
||||
const attemptCount = beforeDrain
|
||||
? Math.max(beforeDrain.attemptCount ?? 0, beforeDrain.retryCount)
|
||||
: 0;
|
||||
// Atomic queue reservation blocks attempt 46. Exhausted rows get an
|
||||
// immediate terminal drain; live retry attempts retain one-second spacing.
|
||||
if (attemptCount < RESTART_NOTICE_MAX_ATTEMPTS) {
|
||||
await waitForRecoveryDrain();
|
||||
}
|
||||
await drainPendingDeliveries({
|
||||
drainKey: `restart-recovery:${params.queueId}`,
|
||||
logLabel: `${params.summary}: restart notice recovery`,
|
||||
cfg: params.cfg,
|
||||
log,
|
||||
deliver: deliverOutboundPayloadsInternal,
|
||||
selectEntry: (entry) => ({
|
||||
match: entry.id === params.queueId,
|
||||
// The caller already waits between attempts. Recovery still reconciles
|
||||
// send-attempt evidence before it permits recipient-visible replay.
|
||||
bypassBackoff: true,
|
||||
}),
|
||||
}).catch((error: unknown) => {
|
||||
log.warn(`${params.summary}: restart notice recovery drain failed: ${String(error)}`, {
|
||||
queueId: params.queueId,
|
||||
sessionKey: params.sessionKey,
|
||||
cycle,
|
||||
});
|
||||
});
|
||||
}
|
||||
const pending = await loadPendingDelivery(params.queueId).catch((error: unknown) => {
|
||||
log.warn(`${params.summary}: restart notice terminal reload failed: ${String(error)}`, {
|
||||
queueId: params.queueId,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
return undefined;
|
||||
});
|
||||
if (pending === null) {
|
||||
return;
|
||||
}
|
||||
log.warn(`${params.summary}: restart notice remains queued after bounded recovery`, {
|
||||
queueId: params.queueId,
|
||||
sessionKey: params.sessionKey,
|
||||
retryCount: pending?.retryCount ?? null,
|
||||
attemptCount: pending?.attemptCount ?? null,
|
||||
maxAttempts: RESTART_NOTICE_MAX_ATTEMPTS,
|
||||
});
|
||||
}
|
||||
|
||||
export async function deliverRestartSentinelNotice(
|
||||
params: RestartSentinelNoticeRoute & {
|
||||
deps: CliDeps;
|
||||
cfg: OpenClawConfig;
|
||||
sessionKey: string;
|
||||
summary: string;
|
||||
message: string;
|
||||
queueId: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
const claim = await withActiveDeliveryClaim(params.queueId, async () => {
|
||||
try {
|
||||
const reservation = await reserveDeliveryAttempt(params.queueId, RESTART_NOTICE_MAX_ATTEMPTS);
|
||||
if (reservation.status === "exhausted") {
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(
|
||||
`${params.summary}: outbound delivery attempt reservation failed; queued for recovery: ${formatErrorMessage(err)}`,
|
||||
{
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
sessionKey: params.sessionKey,
|
||||
},
|
||||
);
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const send = await sendDurableMessageBatch({
|
||||
cfg: params.cfg,
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
accountId: params.accountId,
|
||||
replyToId: params.replyToId,
|
||||
threadId: params.threadId,
|
||||
payloads: [{ text: params.message }],
|
||||
session: buildOutboundSessionContext({ cfg: params.cfg, sessionKey: params.sessionKey }),
|
||||
deps: params.deps,
|
||||
bestEffort: false,
|
||||
skipQueue: true,
|
||||
deliveryQueueId: params.queueId,
|
||||
});
|
||||
if (send.status === "failed" || send.status === "partial_failed") {
|
||||
throw send.error;
|
||||
}
|
||||
const results = send.status === "sent" ? send.results : [];
|
||||
if (results.length === 0) {
|
||||
throw new Error("outbound delivery returned no results");
|
||||
}
|
||||
try {
|
||||
await ackDelivery(params.queueId);
|
||||
return true;
|
||||
} catch (err) {
|
||||
const error = formatErrorMessage(err);
|
||||
await failDeliveryAfterPlatformSend(params.queueId, error).catch(() => undefined);
|
||||
log.warn(`${params.summary}: outbound delivery ack failed; queued for recovery: ${error}`, {
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
// The send path records platform-attempt evidence on this queue row.
|
||||
// Durable recovery owns retries so ambiguous outcomes are reconciled
|
||||
// before another recipient-visible send can begin.
|
||||
const error = formatErrorMessage(err);
|
||||
const permanentRejection = findPlatformMessageRejectedError(err);
|
||||
if (permanentRejection) {
|
||||
try {
|
||||
const pending = await loadPendingDelivery(params.queueId);
|
||||
if (pending) {
|
||||
await failPendingDelivery({
|
||||
id: params.queueId,
|
||||
expectedStatus: "pending",
|
||||
lastError: error,
|
||||
entry: pending,
|
||||
});
|
||||
}
|
||||
} catch (persistError) {
|
||||
log.warn(
|
||||
`${params.summary}: permanent rejection persistence failed; queued for recovery: ${formatErrorMessage(persistError)}`,
|
||||
{
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
sessionKey: params.sessionKey,
|
||||
},
|
||||
);
|
||||
return false;
|
||||
}
|
||||
log.warn(`${params.summary}: outbound delivery permanently rejected: ${error}`, {
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
const recordFailure = isProvenDeliveryNotSentError(err)
|
||||
? failDeliveryBeforePlatformSend
|
||||
: failDelivery;
|
||||
await recordFailure(params.queueId, error).catch(() => undefined);
|
||||
log.warn(`${params.summary}: outbound delivery failed; queued for recovery: ${String(err)}`, {
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
});
|
||||
if (claim.status === "claimed-by-other-owner") {
|
||||
log.info(`${params.summary}: durable restart notice claimed by recovery`, {
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
}
|
||||
const needsRecovery =
|
||||
claim.status === "claimed-by-other-owner" || (claim.status === "claimed" && !claim.value);
|
||||
if (needsRecovery) {
|
||||
await drainFailedRestartSentinelNotice({
|
||||
cfg: params.cfg,
|
||||
queueId: params.queueId,
|
||||
sessionKey: params.sessionKey,
|
||||
summary: params.summary,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -25,16 +25,22 @@ type AdvanceSessionDeliveryAgentRunMock =
|
||||
|
||||
const mocks = vi.hoisted(() => {
|
||||
const state = {
|
||||
queuedSessionDelivery: null as Record<string, unknown> | null,
|
||||
queuedSessionDeliveries: new Map<string, Record<string, unknown>>(),
|
||||
nextSessionDeliveryId: 1,
|
||||
};
|
||||
|
||||
return {
|
||||
resolveSessionAgentId: vi.fn(() => "agent-from-key"),
|
||||
get queuedSessionDelivery() {
|
||||
return state.queuedSessionDelivery;
|
||||
return state.queuedSessionDeliveries.values().next().value ?? null;
|
||||
},
|
||||
set queuedSessionDelivery(value: Record<string, unknown> | null) {
|
||||
state.queuedSessionDelivery = value;
|
||||
state.queuedSessionDeliveries.clear();
|
||||
state.nextSessionDeliveryId = 1;
|
||||
if (value) {
|
||||
state.queuedSessionDeliveries.set("session-delivery-1", value);
|
||||
state.nextSessionDeliveryId = 2;
|
||||
}
|
||||
},
|
||||
dispatchGatewayMethodInProcess: vi.fn<InProcessDispatchMock>(async () => ({
|
||||
status: "ok",
|
||||
@@ -46,6 +52,7 @@ const mocks = vi.hoisted(() => {
|
||||
readRestartSentinel: vi.fn(
|
||||
async (): Promise<RestartSentinel> => ({
|
||||
version: 1,
|
||||
revision: 123,
|
||||
payload: {
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
@@ -60,7 +67,7 @@ const mocks = vi.hoisted(() => {
|
||||
}),
|
||||
),
|
||||
finalizeUpdateRestartSentinelRunningVersion: vi.fn(async () => null),
|
||||
clearRestartSentinel: vi.fn(async () => undefined),
|
||||
clearRestartSentinelIfRevision: vi.fn(async () => true),
|
||||
formatRestartSentinelMessage: vi.fn(() => "restart message"),
|
||||
summarizeRestartSentinel: vi.fn(() => "restart summary"),
|
||||
resolveMainSessionKeyFromConfig: vi.fn(() => "agent:main:main"),
|
||||
@@ -100,14 +107,34 @@ const mocks = vi.hoisted(() => {
|
||||
to: "+15550002",
|
||||
})) as (params?: { to?: string }) => { ok: true; to: string } | { ok: false; error: Error }),
|
||||
deliverOutboundPayloads: vi.fn(async () => [{ channel: "whatsapp", messageId: "msg-1" }]),
|
||||
enqueueDelivery: vi.fn(async () => "queue-1"),
|
||||
enqueueDeliveryOnce: vi.fn(async (_payload: unknown, id: string) => ({ id, created: true })),
|
||||
ackDelivery: vi.fn(async () => {}),
|
||||
failDelivery: vi.fn(async () => {}),
|
||||
failDeliveryAfterPlatformSend: vi.fn(async () => {}),
|
||||
failDeliveryBeforePlatformSend: vi.fn(async () => {}),
|
||||
failPendingDelivery: vi.fn(async () => ({ status: "failed" as const })),
|
||||
loadPendingDelivery: vi.fn(async () => null),
|
||||
drainPendingDeliveries: vi.fn(async () => {}),
|
||||
reserveDeliveryAttempt: vi.fn(async () => ({
|
||||
status: "reserved" as const,
|
||||
attemptCount: 1,
|
||||
})),
|
||||
withActiveDeliveryClaim: vi.fn(async (_id: string, fn: () => Promise<unknown>) => ({
|
||||
status: "claimed" as const,
|
||||
value: await fn(),
|
||||
})),
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
enqueueSessionDelivery: vi.fn(async (payload: Record<string, unknown>) => {
|
||||
state.queuedSessionDelivery = payload;
|
||||
return "session-delivery-1";
|
||||
const existing = [...state.queuedSessionDeliveries.entries()].find(
|
||||
([, entry]) => entry.idempotencyKey === payload.idempotencyKey,
|
||||
);
|
||||
if (existing) {
|
||||
return existing[0];
|
||||
}
|
||||
const id = `session-delivery-${state.nextSessionDeliveryId++}`;
|
||||
state.queuedSessionDeliveries.set(id, payload);
|
||||
return id;
|
||||
}),
|
||||
ackSessionDelivery: vi.fn(async () => {}),
|
||||
advanceSessionDeliveryAgentRun: vi.fn<AdvanceSessionDeliveryAgentRunMock>(async () => {}),
|
||||
@@ -122,7 +149,9 @@ const mocks = vi.hoisted(() => {
|
||||
messageId: "generated-media-transcript",
|
||||
})),
|
||||
removeCronRunContinuationSessionIfIdle: vi.fn(async () => {}),
|
||||
loadPendingSessionDelivery: vi.fn(async () => state.queuedSessionDelivery),
|
||||
loadPendingSessionDelivery: vi.fn(
|
||||
async (id: string) => state.queuedSessionDeliveries.get(id) ?? null,
|
||||
),
|
||||
drainPendingSessionDeliveries: vi.fn(
|
||||
async (params: {
|
||||
logLabel: string;
|
||||
@@ -130,7 +159,14 @@ const mocks = vi.hoisted(() => {
|
||||
selectEntry: (entry: Record<string, unknown>, now: number) => { match: boolean };
|
||||
deliver: (entry: Record<string, unknown>) => Promise<void>;
|
||||
}) => {
|
||||
if (!state.queuedSessionDelivery) {
|
||||
const selected = [...state.queuedSessionDeliveries.entries()]
|
||||
.map(([id, payload]) => ({ id, payload }))
|
||||
.find(
|
||||
({ id, payload }) =>
|
||||
params.selectEntry({ id, enqueuedAt: 1, retryCount: 0, ...payload }, Date.now())
|
||||
.match,
|
||||
);
|
||||
if (!selected) {
|
||||
return;
|
||||
}
|
||||
const entry: Record<string, unknown> & {
|
||||
@@ -138,18 +174,14 @@ const mocks = vi.hoisted(() => {
|
||||
enqueuedAt: number;
|
||||
retryCount: number;
|
||||
} = {
|
||||
id: "session-delivery-1",
|
||||
id: selected.id,
|
||||
enqueuedAt: 1,
|
||||
retryCount: 0,
|
||||
...state.queuedSessionDelivery,
|
||||
...selected.payload,
|
||||
};
|
||||
const decision = params.selectEntry(entry, Date.now());
|
||||
if (!decision.match) {
|
||||
return;
|
||||
}
|
||||
const maxRetries = typeof entry["maxRetries"] === "number" ? entry["maxRetries"] : 5;
|
||||
if (entry.retryCount >= maxRetries) {
|
||||
state.queuedSessionDelivery = null;
|
||||
state.queuedSessionDeliveries.delete(entry.id);
|
||||
params.log.warn(
|
||||
`${params.logLabel}: entry ${entry.id} exceeded max retries and was moved to failed/`,
|
||||
);
|
||||
@@ -157,13 +189,13 @@ const mocks = vi.hoisted(() => {
|
||||
}
|
||||
try {
|
||||
await params.deliver(entry);
|
||||
state.queuedSessionDelivery = null;
|
||||
state.queuedSessionDeliveries.delete(entry.id);
|
||||
} catch (err) {
|
||||
state.queuedSessionDelivery = {
|
||||
state.queuedSessionDeliveries.set(entry.id, {
|
||||
...entry,
|
||||
retryCount: entry.retryCount + 1,
|
||||
lastError: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
});
|
||||
params.log.warn(`${params.logLabel}: retry failed for entry ${entry.id}: ${String(err)}`);
|
||||
}
|
||||
},
|
||||
@@ -215,7 +247,7 @@ vi.mock("../agents/agent-scope.js", async () => {
|
||||
vi.mock("../infra/restart-sentinel.js", () => ({
|
||||
finalizeUpdateRestartSentinelRunningVersion: mocks.finalizeUpdateRestartSentinelRunningVersion,
|
||||
readRestartSentinel: mocks.readRestartSentinel,
|
||||
clearRestartSentinel: mocks.clearRestartSentinel,
|
||||
clearRestartSentinelIfRevision: mocks.clearRestartSentinelIfRevision,
|
||||
formatRestartSentinelMessage: mocks.formatRestartSentinelMessage,
|
||||
summarizeRestartSentinel: mocks.summarizeRestartSentinel,
|
||||
}));
|
||||
@@ -321,9 +353,19 @@ vi.mock("../infra/outbound/deliver.js", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("../infra/outbound/delivery-queue.js", () => ({
|
||||
enqueueDelivery: mocks.enqueueDelivery,
|
||||
enqueueDeliveryOnce: mocks.enqueueDeliveryOnce,
|
||||
ackDelivery: mocks.ackDelivery,
|
||||
failDelivery: mocks.failDelivery,
|
||||
failDeliveryAfterPlatformSend: mocks.failDeliveryAfterPlatformSend,
|
||||
failDeliveryBeforePlatformSend: mocks.failDeliveryBeforePlatformSend,
|
||||
drainPendingDeliveries: mocks.drainPendingDeliveries,
|
||||
withActiveDeliveryClaim: mocks.withActiveDeliveryClaim,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/outbound/delivery-queue-storage.js", () => ({
|
||||
failPendingDelivery: mocks.failPendingDelivery,
|
||||
loadPendingDelivery: mocks.loadPendingDelivery,
|
||||
reserveDeliveryAttempt: mocks.reserveDeliveryAttempt,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/system-events.js", () => ({
|
||||
@@ -452,6 +494,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
mocks.readRestartSentinel.mockReset();
|
||||
mocks.readRestartSentinel.mockResolvedValue({
|
||||
version: 1,
|
||||
revision: 123,
|
||||
payload: {
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
@@ -488,10 +531,18 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
mocks.resolveOutboundTarget.mockReturnValue({ ok: true as const, to: "+15550002" });
|
||||
mocks.deliverOutboundPayloads.mockReset();
|
||||
mocks.deliverOutboundPayloads.mockResolvedValue([{ channel: "whatsapp", messageId: "msg-1" }]);
|
||||
mocks.enqueueDelivery.mockReset();
|
||||
mocks.enqueueDelivery.mockResolvedValue("queue-1");
|
||||
mocks.enqueueDeliveryOnce.mockReset();
|
||||
mocks.enqueueDeliveryOnce.mockImplementation(async (_payload, id) => ({ id, created: true }));
|
||||
mocks.ackDelivery.mockClear();
|
||||
mocks.failDelivery.mockClear();
|
||||
mocks.failDeliveryAfterPlatformSend.mockClear();
|
||||
mocks.failDeliveryBeforePlatformSend.mockClear();
|
||||
mocks.failPendingDelivery.mockClear();
|
||||
mocks.loadPendingDelivery.mockReset();
|
||||
mocks.loadPendingDelivery.mockResolvedValue(null);
|
||||
mocks.drainPendingDeliveries.mockClear();
|
||||
mocks.reserveDeliveryAttempt.mockClear();
|
||||
mocks.withActiveDeliveryClaim.mockClear();
|
||||
mocks.enqueueSystemEvent.mockClear();
|
||||
mocks.requestHeartbeat.mockClear();
|
||||
mocks.enqueueSessionDelivery.mockClear();
|
||||
@@ -509,7 +560,8 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
mocks.recoverPendingSessionDeliveries.mockClear();
|
||||
mocks.finalizeUpdateRestartSentinelRunningVersion.mockReset();
|
||||
mocks.finalizeUpdateRestartSentinelRunningVersion.mockResolvedValue(null);
|
||||
mocks.clearRestartSentinel.mockClear();
|
||||
mocks.clearRestartSentinelIfRevision.mockReset();
|
||||
mocks.clearRestartSentinelIfRevision.mockResolvedValue(true);
|
||||
mocks.formatRestartSentinelMessage.mockClear();
|
||||
mocks.summarizeRestartSentinel.mockClear();
|
||||
mocks.injectTimestamp.mockClear();
|
||||
@@ -533,14 +585,21 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
deps,
|
||||
bestEffort: false,
|
||||
skipQueue: true,
|
||||
deliveryQueueId: "restart-sentinel-notice:agent:main:main:123",
|
||||
});
|
||||
expectMockCallFields(mocks.enqueueDelivery, {
|
||||
expectMockCallFields(mocks.enqueueDeliveryOnce, {
|
||||
channel: "whatsapp",
|
||||
to: "+15550002",
|
||||
payloads: [{ text: "restart message" }],
|
||||
bestEffort: false,
|
||||
completionRetention: "permanent",
|
||||
maxRetries: 45,
|
||||
});
|
||||
expect(mocks.ackDelivery).toHaveBeenCalledWith("queue-1");
|
||||
expect(mocks.ackDelivery).toHaveBeenCalledWith("restart-sentinel-notice:agent:main:main:123");
|
||||
expect(mocks.reserveDeliveryAttempt).toHaveBeenCalledWith(
|
||||
"restart-sentinel-notice:agent:main:main:123",
|
||||
45,
|
||||
);
|
||||
expect(mocks.failDelivery).not.toHaveBeenCalled();
|
||||
expect(mocks.formatRestartSentinelMessage).toHaveBeenCalledWith(expect.anything());
|
||||
expect(mocks.summarizeRestartSentinel).toHaveBeenCalledWith(expect.anything());
|
||||
@@ -558,60 +617,263 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
expect(mocks.logWarn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries outbound delivery once and logs a warning without dropping the agent wake", async () => {
|
||||
vi.useFakeTimers();
|
||||
mocks.deliverOutboundPayloads
|
||||
.mockRejectedValueOnce(new Error("transport not ready"))
|
||||
.mockResolvedValueOnce([{ channel: "whatsapp", messageId: "msg-2" }]);
|
||||
it("persists every downstream intent before consuming the loaded revision", async () => {
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
const wakePromise = scheduleRestartSentinelWake({ deps: {} as never });
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await wakePromise;
|
||||
expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123);
|
||||
const clearOrder = mocks.clearRestartSentinelIfRevision.mock.invocationCallOrder[0] ?? 0;
|
||||
expect(mocks.enqueueSessionDelivery.mock.invocationCallOrder[0]).toBeLessThan(clearOrder);
|
||||
expect(mocks.enqueueDeliveryOnce.mock.invocationCallOrder[0]).toBeLessThan(clearOrder);
|
||||
expect(clearOrder).toBeLessThan(mocks.enqueueSystemEvent.mock.invocationCallOrder[0] ?? 0);
|
||||
expect(clearOrder).toBeLessThan(mocks.deliverOutboundPayloads.mock.invocationCallOrder[0] ?? 0);
|
||||
});
|
||||
|
||||
expect(mocks.enqueueDelivery).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledTimes(2);
|
||||
expectMockCallFields(mocks.deliverOutboundPayloads, { skipQueue: true }, 0);
|
||||
expectMockCallFields(mocks.deliverOutboundPayloads, { skipQueue: true }, 1);
|
||||
expect(mocks.ackDelivery).toHaveBeenCalledWith("queue-1");
|
||||
it("stops delivery when guarded sentinel consumption fails", async () => {
|
||||
mocks.clearRestartSentinelIfRevision.mockRejectedValueOnce(new Error("database locked"));
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.enqueueSessionDelivery).toHaveBeenCalledOnce();
|
||||
expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledOnce();
|
||||
expect(mocks.enqueueSystemEvent).not.toHaveBeenCalled();
|
||||
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
|
||||
expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", {
|
||||
source: "restart-sentinel",
|
||||
sessionKey: "agent:main:main",
|
||||
reason: "database locked",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a newer sentinel while draining durable work from the loaded revision", async () => {
|
||||
mocks.clearRestartSentinelIfRevision.mockResolvedValueOnce(false);
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123);
|
||||
expect(mocks.enqueueSystemEvent).toHaveBeenCalledOnce();
|
||||
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce();
|
||||
expect(mocks.logInfo).toHaveBeenCalledWith(
|
||||
"restart summary: newer restart sentinel preserved while draining durable work",
|
||||
{ sessionKey: "agent:main:main" },
|
||||
);
|
||||
});
|
||||
|
||||
it("does not resend a restart notice whose stable queue id is already owned", async () => {
|
||||
mocks.enqueueDeliveryOnce.mockImplementationOnce(async (_payload, id) => ({
|
||||
id,
|
||||
created: false,
|
||||
}));
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledWith(123);
|
||||
expect(mocks.enqueueDeliveryOnce.mock.calls[0]?.[1]).toBe(
|
||||
"restart-sentinel-notice:agent:main:main:123",
|
||||
);
|
||||
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
|
||||
expect(mocks.ackDelivery).not.toHaveBeenCalled();
|
||||
expect(mocks.failDelivery).not.toHaveBeenCalled();
|
||||
expect(mocks.enqueueSystemEvent).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.logWarn.mock.calls).toEqual([
|
||||
[
|
||||
"restart summary: outbound delivery failed; retrying in 1000ms: Error: transport not ready",
|
||||
{
|
||||
channel: "whatsapp",
|
||||
to: "+15550002",
|
||||
sessionKey: "agent:main:main",
|
||||
attempt: 1,
|
||||
maxAttempts: 45,
|
||||
},
|
||||
],
|
||||
expect(mocks.logInfo).toHaveBeenCalledWith(
|
||||
"restart summary: durable restart notice already owned",
|
||||
{ sessionKey: "agent:main:main" },
|
||||
);
|
||||
});
|
||||
|
||||
it("queues the restart wake before a system-event continuation", async () => {
|
||||
mocks.readRestartSentinel.mockResolvedValueOnce({
|
||||
version: 1,
|
||||
revision: 123,
|
||||
payload: {
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 99,
|
||||
sessionKey: "agent:main:main",
|
||||
continuation: { kind: "systemEvent", text: "continue" },
|
||||
},
|
||||
});
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.enqueueSessionDelivery).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.enqueueSessionDelivery).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
text: "restart message",
|
||||
idempotencyKey: "restart-sentinel-wake:agent:main:main:123",
|
||||
completionRetention: "permanent",
|
||||
}),
|
||||
);
|
||||
expect(mocks.enqueueSessionDelivery).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
text: "continue",
|
||||
idempotencyKey: "restart-sentinel:agent:main:main:systemEvent:123",
|
||||
completionRetention: "permanent",
|
||||
}),
|
||||
);
|
||||
expect(mocks.enqueueSystemEvent.mock.calls.map((call) => call[0])).toEqual([
|
||||
"restart message",
|
||||
"continue",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps one queued restart notice when outbound retries are exhausted", async () => {
|
||||
vi.useFakeTimers();
|
||||
mocks.deliverOutboundPayloads.mockRejectedValue(new Error("transport still not ready"));
|
||||
it("queues a failed outbound notice for durable recovery without dropping the agent wake", async () => {
|
||||
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("platform outcome unknown"));
|
||||
mocks.loadPendingDelivery
|
||||
.mockResolvedValueOnce({
|
||||
id: "restart-sentinel-notice:agent:main:main:123",
|
||||
retryCount: 1,
|
||||
lastError: "platform outcome unknown",
|
||||
} as never)
|
||||
.mockResolvedValue(null);
|
||||
|
||||
const wakePromise = scheduleRestartSentinelWake({ deps: {} as never });
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await vi.advanceTimersByTimeAsync(44_000);
|
||||
await wakePromise;
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.enqueueDelivery).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledTimes(45);
|
||||
expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce();
|
||||
expectMockCallFields(mocks.deliverOutboundPayloads, {
|
||||
skipQueue: true,
|
||||
deliveryQueueId: "restart-sentinel-notice:agent:main:main:123",
|
||||
});
|
||||
expect(mocks.ackDelivery).not.toHaveBeenCalled();
|
||||
expect(mocks.failDelivery).toHaveBeenCalledWith("queue-1", "transport still not ready");
|
||||
expect(mocks.failDelivery).toHaveBeenCalledWith(
|
||||
"restart-sentinel-notice:agent:main:main:123",
|
||||
"platform outcome unknown",
|
||||
);
|
||||
expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce();
|
||||
const drain = expectRecordFields(mockCallArg(mocks.drainPendingDeliveries), {
|
||||
drainKey: "restart-recovery:restart-sentinel-notice:agent:main:main:123",
|
||||
deliver: expect.any(Function),
|
||||
});
|
||||
const selectEntry = drain.selectEntry as (entry: { id: string }) => {
|
||||
match: boolean;
|
||||
bypassBackoff?: boolean;
|
||||
};
|
||||
expect(selectEntry({ id: "restart-sentinel-notice:agent:main:main:123" })).toEqual({
|
||||
match: true,
|
||||
bypassBackoff: true,
|
||||
});
|
||||
expect(selectEntry({ id: "other" })).toEqual({ match: false, bypassBackoff: true });
|
||||
expect(mocks.enqueueSystemEvent).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.logWarn).toHaveBeenCalledWith(
|
||||
"restart summary: outbound delivery failed; queued for recovery: Error: platform outcome unknown",
|
||||
{
|
||||
channel: "whatsapp",
|
||||
to: "+15550002",
|
||||
sessionKey: "agent:main:main",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("still dispatches continuation after restart notice retries are exhausted", async () => {
|
||||
vi.useFakeTimers();
|
||||
mocks.deliverOutboundPayloads.mockRejectedValue(new Error("transport still not ready"));
|
||||
it("starts a terminal drain when the persisted retry budget is exhausted", async () => {
|
||||
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport unavailable"));
|
||||
mocks.loadPendingDelivery.mockResolvedValue({
|
||||
id: "restart-sentinel-notice:agent:main:main:123",
|
||||
retryCount: 45,
|
||||
attemptCount: 45,
|
||||
lastError: "transport unavailable",
|
||||
} as never);
|
||||
mocks.drainPendingDeliveries.mockImplementationOnce(async () => {
|
||||
mocks.loadPendingDelivery.mockResolvedValue(null);
|
||||
});
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("preserves a notice whose persisted retry budget is not exhausted", async () => {
|
||||
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport unavailable"));
|
||||
mocks.loadPendingDelivery.mockResolvedValue({
|
||||
id: "restart-sentinel-notice:agent:main:main:123",
|
||||
retryCount: 7,
|
||||
lastError: "database busy",
|
||||
} as never);
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.drainPendingDeliveries).toHaveBeenCalledTimes(46);
|
||||
expect(mocks.failPendingDelivery).not.toHaveBeenCalled();
|
||||
expect(mocks.logWarn).toHaveBeenCalledWith(
|
||||
"restart summary: restart notice remains queued after bounded recovery",
|
||||
{
|
||||
queueId: "restart-sentinel-notice:agent:main:main:123",
|
||||
sessionKey: "agent:main:main",
|
||||
retryCount: 7,
|
||||
attemptCount: null,
|
||||
maxAttempts: 45,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("continues exact-id recovery after another owner releases the notice", async () => {
|
||||
mocks.withActiveDeliveryClaim.mockResolvedValueOnce({
|
||||
status: "claimed-by-other-owner",
|
||||
} as never);
|
||||
mocks.loadPendingDelivery
|
||||
.mockResolvedValueOnce({
|
||||
id: "restart-sentinel-notice:agent:main:main:123",
|
||||
retryCount: 0,
|
||||
} as never)
|
||||
.mockResolvedValue(null);
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
|
||||
expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce();
|
||||
expect(mocks.logInfo).toHaveBeenCalledWith(
|
||||
"restart summary: durable restart notice claimed by recovery",
|
||||
{ sessionKey: "agent:main:main" },
|
||||
);
|
||||
});
|
||||
|
||||
it("schedules safe recovery when the delivered notice cannot be acknowledged", async () => {
|
||||
mocks.ackDelivery.mockRejectedValueOnce(new Error("ack unavailable"));
|
||||
mocks.loadPendingDelivery
|
||||
.mockResolvedValueOnce({
|
||||
id: "restart-sentinel-notice:agent:main:main:123",
|
||||
retryCount: 1,
|
||||
recoveryState: "unknown_after_send",
|
||||
} as never)
|
||||
.mockResolvedValue(null);
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.failDeliveryAfterPlatformSend).toHaveBeenCalledWith(
|
||||
"restart-sentinel-notice:agent:main:main:123",
|
||||
"ack unavailable",
|
||||
);
|
||||
expect(mocks.drainPendingDeliveries).toHaveBeenCalledOnce();
|
||||
expect(mocks.logWarn).toHaveBeenCalledWith(
|
||||
"restart summary: outbound delivery ack failed; queued for recovery: ack unavailable",
|
||||
{
|
||||
channel: "whatsapp",
|
||||
to: "+15550002",
|
||||
sessionKey: "agent:main:main",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps one queued restart notice when outbound delivery fails", async () => {
|
||||
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport still not ready"));
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.enqueueDeliveryOnce).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.deliverOutboundPayloads).toHaveBeenCalledOnce();
|
||||
expect(mocks.ackDelivery).not.toHaveBeenCalled();
|
||||
expect(mocks.failDelivery).toHaveBeenCalledWith(
|
||||
"restart-sentinel-notice:agent:main:main:123",
|
||||
"transport still not ready",
|
||||
);
|
||||
});
|
||||
|
||||
it("still dispatches continuation after a restart notice is queued for recovery", async () => {
|
||||
mocks.deliverOutboundPayloads.mockRejectedValueOnce(new Error("transport still not ready"));
|
||||
mocks.readRestartSentinel.mockResolvedValue({
|
||||
version: 1,
|
||||
revision: 123,
|
||||
payload: {
|
||||
sessionKey: "agent:main:main",
|
||||
deliveryContext: {
|
||||
@@ -627,13 +889,12 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
},
|
||||
} as unknown as Awaited<ReturnType<typeof mocks.readRestartSentinel>>);
|
||||
|
||||
const wakePromise = scheduleRestartSentinelWake({ deps: {} as never });
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await vi.advanceTimersByTimeAsync(44_000);
|
||||
await wakePromise;
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.failDelivery).toHaveBeenCalledWith("queue-1", "transport still not ready");
|
||||
expect(mocks.failDelivery).toHaveBeenCalledWith(
|
||||
"restart-sentinel-notice:agent:main:main:123",
|
||||
"transport still not ready",
|
||||
);
|
||||
expect(mocks.recordInboundSessionAndDispatchReply).toHaveBeenCalledTimes(1);
|
||||
expectContinuationDispatchFields({ routeSessionKey: "agent:main:main" }, { Body: "continue" });
|
||||
});
|
||||
@@ -693,7 +954,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expectMockCallFields(mocks.enqueueDelivery, {
|
||||
expectMockCallFields(mocks.enqueueDeliveryOnce, {
|
||||
payloads: [{ text: "restart message" }],
|
||||
threadId: "thread-42",
|
||||
});
|
||||
@@ -2114,6 +2375,8 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
|
||||
it("dispatches agentTurn continuation for a completed run entry", async () => {
|
||||
mocks.readRestartSentinel.mockResolvedValue({
|
||||
version: 1,
|
||||
revision: 123,
|
||||
payload: {
|
||||
sessionKey: "agent:main:main",
|
||||
deliveryContext: {
|
||||
@@ -2153,6 +2416,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
message: "continue after restart",
|
||||
messageId: "restart-sentinel:agent:main:main:agentTurn:123",
|
||||
expectedSessionId: "agent:main:main",
|
||||
completionRetention: "permanent",
|
||||
route: {
|
||||
channel: "whatsapp",
|
||||
to: "+15550002",
|
||||
@@ -2649,6 +2913,8 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
const busyReply = "⚠️ Previous run is still shutting down. Please try again in a moment.";
|
||||
let attempt = 0;
|
||||
mocks.readRestartSentinel.mockResolvedValue({
|
||||
version: 1,
|
||||
revision: 123,
|
||||
payload: {
|
||||
sessionKey: "agent:main:main",
|
||||
deliveryContext: {
|
||||
@@ -2766,7 +3032,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.clearRestartSentinel).not.toHaveBeenCalled();
|
||||
expect(mocks.clearRestartSentinelIfRevision).not.toHaveBeenCalled();
|
||||
expect(mocks.drainPendingSessionDeliveries).not.toHaveBeenCalled();
|
||||
expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", {
|
||||
source: "restart-sentinel",
|
||||
@@ -2825,12 +3091,13 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
};
|
||||
mocks.readRestartSentinel.mockResolvedValue({
|
||||
version: 1,
|
||||
revision: 123,
|
||||
payload,
|
||||
});
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce();
|
||||
expect(mocks.clearRestartSentinelIfRevision).toHaveBeenCalledOnce();
|
||||
expect(getLatestUpdateRestartSentinel()).toEqual(payload);
|
||||
});
|
||||
|
||||
@@ -2847,6 +3114,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
};
|
||||
mocks.readRestartSentinel.mockResolvedValue({
|
||||
version: 1,
|
||||
revision: 123,
|
||||
payload,
|
||||
});
|
||||
|
||||
@@ -2856,7 +3124,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
expect(getLatestUpdateRestartSentinel()).toEqual(payload);
|
||||
});
|
||||
|
||||
it("does not wake the main session when the sentinel has no sessionKey", async () => {
|
||||
it("durably wakes the main session when the sentinel has no sessionKey", async () => {
|
||||
mocks.readRestartSentinel.mockResolvedValue({
|
||||
payload: {
|
||||
message: "restart message",
|
||||
@@ -2868,7 +3136,12 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
expect(mocks.enqueueSystemEvent).toHaveBeenCalledWith("restart message", {
|
||||
sessionKey: "agent:main:main",
|
||||
});
|
||||
expect(mocks.requestHeartbeat).not.toHaveBeenCalled();
|
||||
expect(mocks.requestHeartbeat).toHaveBeenCalledWith({
|
||||
source: "restart-sentinel",
|
||||
intent: "immediate",
|
||||
reason: "wake",
|
||||
sessionKey: "agent:main:main",
|
||||
});
|
||||
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -2910,6 +3183,15 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
threadId: undefined,
|
||||
});
|
||||
mocks.deliveryContextFromSession.mockReturnValue(undefined);
|
||||
mocks.loadSessionEntry.mockReturnValue({
|
||||
cfg: {},
|
||||
entry: { sessionId: "agent:main:matrix:channel:!lowercased:example.org", updatedAt: 0 },
|
||||
store: {},
|
||||
storePath: "/tmp/sessions.json",
|
||||
canonicalKey: "agent:main:matrix:channel:!lowercased:example.org",
|
||||
storeKeys: ["agent:main:matrix:channel:!lowercased:example.org"],
|
||||
legacyKey: undefined,
|
||||
});
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
@@ -2918,7 +3200,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
sessionKey: "agent:main:matrix:channel:!lowercased:example.org",
|
||||
});
|
||||
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
|
||||
expect(mocks.enqueueDelivery).not.toHaveBeenCalled();
|
||||
expect(mocks.enqueueDeliveryOnce).not.toHaveBeenCalled();
|
||||
expect(mocks.resolveOutboundTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import { REPLY_RUN_STILL_SHUTTING_DOWN_TEXT } from "../auto-reply/reply/get-repl
|
||||
import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
|
||||
import { dispatchReplyWithBufferedBlockDispatcher } from "../auto-reply/reply/provider-dispatcher.js";
|
||||
import type { ChatType } from "../channels/chat-type.js";
|
||||
import { sendDurableMessageBatch } from "../channels/message/runtime.js";
|
||||
import { getChannelPlugin, normalizeChannelId } from "../channels/plugins/index.js";
|
||||
import { recordInboundSession } from "../channels/session.js";
|
||||
import { dispatchAssembledChannelTurn } from "../channels/turn/kernel.js";
|
||||
@@ -14,11 +13,9 @@ import { resolveMainSessionKeyFromConfig } from "../config/sessions.js";
|
||||
import { parseSessionThreadInfo } from "../config/sessions/thread-info.js";
|
||||
import { formatErrorMessage, toErrorObject } from "../infra/errors.js";
|
||||
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
|
||||
import { ackDelivery, enqueueDelivery, failDelivery } from "../infra/outbound/delivery-queue.js";
|
||||
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
|
||||
import { resolveOutboundTarget } from "../infra/outbound/targets.js";
|
||||
import {
|
||||
clearRestartSentinel,
|
||||
clearRestartSentinelIfRevision,
|
||||
finalizeUpdateRestartSentinelRunningVersion,
|
||||
formatRestartSentinelMessage,
|
||||
readRestartSentinel,
|
||||
@@ -53,12 +50,14 @@ import {
|
||||
} from "../utils/delivery-context.shared.js";
|
||||
import { INTERNAL_MESSAGE_CHANNEL } from "../utils/message-channel.js";
|
||||
import { deliverQueuedGeneratedMediaAgentTurn } from "./server-restart-sentinel-agent-delivery.js";
|
||||
import {
|
||||
deliverRestartSentinelNotice,
|
||||
enqueueRestartSentinelNotice,
|
||||
} from "./server-restart-sentinel-notice.js";
|
||||
import { loadSessionEntry } from "./session-utils.js";
|
||||
import { runStartupTasks, type StartupTask } from "./startup-tasks.js";
|
||||
|
||||
const log = createSubsystemLogger("gateway/restart-sentinel");
|
||||
const OUTBOUND_RETRY_DELAY_MS = 1_000;
|
||||
const OUTBOUND_MAX_ATTEMPTS = 45;
|
||||
const RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS = process.env.VITEST ? 1 : 6_000;
|
||||
const RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS = 20;
|
||||
const CONTROL_PLANE_UPDATE_PENDING_RETRY_DELAY_MS = process.env.VITEST ? 1 : 2_000;
|
||||
@@ -106,89 +105,19 @@ function enqueueRestartSentinelWake(
|
||||
requestHeartbeat({ source: "restart-sentinel", intent: "immediate", reason: "wake", sessionKey });
|
||||
}
|
||||
|
||||
async function waitForOutboundRetry(delayMs: number) {
|
||||
async function waitForRetry(delayMs: number) {
|
||||
await new Promise<void>((resolve) => {
|
||||
const timer = setTimeout(resolve, delayMs);
|
||||
timer.unref?.();
|
||||
});
|
||||
}
|
||||
|
||||
async function deliverRestartSentinelNotice(params: {
|
||||
deps: CliDeps;
|
||||
cfg: ReturnType<typeof loadSessionEntry>["cfg"];
|
||||
sessionKey: string;
|
||||
summary: string;
|
||||
message: string;
|
||||
channel: string;
|
||||
to: string;
|
||||
accountId?: string;
|
||||
replyToId?: string;
|
||||
threadId?: string;
|
||||
session: ReturnType<typeof buildOutboundSessionContext>;
|
||||
}) {
|
||||
const payloads = [{ text: params.message }];
|
||||
const queueId = await enqueueDelivery({
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
accountId: params.accountId,
|
||||
replyToId: params.replyToId,
|
||||
threadId: params.threadId,
|
||||
payloads,
|
||||
bestEffort: false,
|
||||
}).catch(() => null);
|
||||
for (let attempt = 1; attempt <= OUTBOUND_MAX_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
const send = await sendDurableMessageBatch({
|
||||
cfg: params.cfg,
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
accountId: params.accountId,
|
||||
replyToId: params.replyToId,
|
||||
threadId: params.threadId,
|
||||
payloads,
|
||||
session: params.session,
|
||||
deps: params.deps,
|
||||
bestEffort: false,
|
||||
skipQueue: true,
|
||||
});
|
||||
if (send.status === "failed" || send.status === "partial_failed") {
|
||||
throw send.error;
|
||||
}
|
||||
const results = send.status === "sent" ? send.results : [];
|
||||
if (results.length > 0) {
|
||||
if (queueId) {
|
||||
await ackDelivery(queueId).catch(() => {});
|
||||
}
|
||||
return;
|
||||
}
|
||||
throw new Error("outbound delivery returned no results");
|
||||
} catch (err) {
|
||||
const retrying = attempt < OUTBOUND_MAX_ATTEMPTS;
|
||||
const suffix = retrying ? `; retrying in ${OUTBOUND_RETRY_DELAY_MS}ms` : "";
|
||||
log.warn(`${params.summary}: outbound delivery failed${suffix}: ${String(err)}`, {
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
sessionKey: params.sessionKey,
|
||||
attempt,
|
||||
maxAttempts: OUTBOUND_MAX_ATTEMPTS,
|
||||
});
|
||||
if (!retrying) {
|
||||
if (queueId) {
|
||||
await failDelivery(queueId, formatErrorMessage(err)).catch(() => undefined);
|
||||
}
|
||||
return;
|
||||
}
|
||||
await waitForOutboundRetry(OUTBOUND_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function buildRestartContinuationMessageId(params: {
|
||||
sessionKey: string;
|
||||
kind: RestartSentinelContinuation["kind"];
|
||||
ts: number;
|
||||
revision: number;
|
||||
}) {
|
||||
return `restart-sentinel:${params.sessionKey}:${params.kind}:${params.ts}`;
|
||||
return `restart-sentinel:${params.sessionKey}:${params.kind}:${params.revision}`;
|
||||
}
|
||||
|
||||
function resolveRestartContinuationRoute(params: {
|
||||
@@ -405,19 +334,22 @@ function buildQueuedRestartContinuation(params: {
|
||||
continuation: RestartSentinelContinuation;
|
||||
route?: SessionDeliveryRoute;
|
||||
expectedSessionId?: string | undefined;
|
||||
ts: number;
|
||||
revision: number;
|
||||
deliveryContext?: {
|
||||
channel?: string;
|
||||
to?: string;
|
||||
accountId?: string;
|
||||
threadId?: string | number;
|
||||
};
|
||||
idempotencyKey?: string;
|
||||
}): QueuedSessionDeliveryPayload {
|
||||
const idempotencyKey = buildRestartContinuationMessageId({
|
||||
sessionKey: params.sessionKey,
|
||||
kind: params.continuation.kind,
|
||||
ts: params.ts,
|
||||
});
|
||||
const idempotencyKey =
|
||||
params.idempotencyKey ??
|
||||
buildRestartContinuationMessageId({
|
||||
sessionKey: params.sessionKey,
|
||||
kind: params.continuation.kind,
|
||||
revision: params.revision,
|
||||
});
|
||||
if (params.continuation.kind === "systemEvent") {
|
||||
return {
|
||||
kind: "systemEvent",
|
||||
@@ -426,6 +358,7 @@ function buildQueuedRestartContinuation(params: {
|
||||
...(params.deliveryContext ? { deliveryContext: params.deliveryContext } : {}),
|
||||
idempotencyKey,
|
||||
maxRetries: RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS,
|
||||
completionRetention: "permanent",
|
||||
};
|
||||
}
|
||||
return {
|
||||
@@ -435,6 +368,7 @@ function buildQueuedRestartContinuation(params: {
|
||||
messageId: idempotencyKey,
|
||||
...(params.expectedSessionId ? { expectedSessionId: params.expectedSessionId } : {}),
|
||||
maxRetries: RESTART_CONTINUATION_BUSY_MAX_ATTEMPTS,
|
||||
completionRetention: "permanent",
|
||||
...(params.route ? { route: params.route } : {}),
|
||||
...(params.deliveryContext ? { deliveryContext: params.deliveryContext } : {}),
|
||||
idempotencyKey,
|
||||
@@ -474,7 +408,7 @@ async function drainRestartContinuationQueue(params: {
|
||||
params.log.info(
|
||||
`restart continuation: entry ${params.entryId} still waiting for the previous run to clear; retrying in ${RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS}ms`,
|
||||
);
|
||||
await waitForOutboundRetry(RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS);
|
||||
await waitForRetry(RESTART_CONTINUATION_BUSY_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -505,6 +439,7 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
return null;
|
||||
}
|
||||
const payload = sentinel.payload;
|
||||
const sentinelRevision = sentinel.revision;
|
||||
if (payload.kind === "update") {
|
||||
recordLatestUpdateRestartSentinel(payload);
|
||||
}
|
||||
@@ -543,14 +478,25 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
|
||||
if (!sessionKey) {
|
||||
const mainSessionKey = resolveMainSessionKeyFromConfig();
|
||||
enqueueSystemEvent(message, { sessionKey: mainSessionKey });
|
||||
const wakeQueueId = await enqueueSessionDelivery(
|
||||
buildQueuedRestartContinuation({
|
||||
sessionKey: mainSessionKey,
|
||||
continuation: { kind: "systemEvent", text: message },
|
||||
revision: sentinelRevision,
|
||||
idempotencyKey: `restart-sentinel-wake:${mainSessionKey}:${sentinelRevision}`,
|
||||
}),
|
||||
);
|
||||
if (payload.continuation) {
|
||||
log.warn(`${summary}: continuation skipped: restart sentinel sessionKey unavailable`, {
|
||||
sessionKey: mainSessionKey,
|
||||
continuationKind: payload.continuation.kind,
|
||||
});
|
||||
}
|
||||
await clearRestartSentinel();
|
||||
const consumed = await clearRestartSentinelIfRevision(sentinelRevision);
|
||||
if (!consumed) {
|
||||
log.info(`${summary}: newer restart sentinel preserved while draining durable wake`);
|
||||
}
|
||||
await drainRestartContinuationQueue({ deps: params.deps, entryId: wakeQueueId, log });
|
||||
return { status: "ran" as const };
|
||||
}
|
||||
|
||||
@@ -587,6 +533,9 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
let replyToId: string | undefined;
|
||||
let resolvedThreadId = threadId;
|
||||
let continuationQueueId: string | undefined;
|
||||
let wakeQueueId: string | undefined;
|
||||
let noticeQueueId: string | undefined;
|
||||
let noticeQueueCreated = false;
|
||||
let continuationRoute: SessionDeliveryRoute | undefined;
|
||||
|
||||
if (channel && to) {
|
||||
@@ -624,11 +573,28 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
threadId: resolvedThreadId,
|
||||
chatType,
|
||||
});
|
||||
}
|
||||
|
||||
const routedAgentTurnContinuation =
|
||||
payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined;
|
||||
if (!routedAgentTurnContinuation) {
|
||||
wakeQueueId = await enqueueSessionDelivery(
|
||||
buildQueuedRestartContinuation({
|
||||
sessionKey: canonicalKey,
|
||||
continuation: { kind: "systemEvent", text: message },
|
||||
revision: sentinelRevision,
|
||||
deliveryContext: wakeDeliveryContext,
|
||||
idempotencyKey: `restart-sentinel-wake:${canonicalKey}:${sentinelRevision}`,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (payload.continuation) {
|
||||
continuationQueueId = await enqueueSessionDelivery(
|
||||
buildQueuedRestartContinuation({
|
||||
sessionKey: canonicalKey,
|
||||
continuation: payload.continuation,
|
||||
ts: payload.ts,
|
||||
revision: sentinelRevision,
|
||||
route: continuationRoute,
|
||||
expectedSessionId: entry?.sessionId,
|
||||
deliveryContext:
|
||||
@@ -644,19 +610,35 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
);
|
||||
}
|
||||
|
||||
await clearRestartSentinel();
|
||||
const routedAgentTurnContinuation =
|
||||
payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined;
|
||||
if (!routedAgentTurnContinuation) {
|
||||
enqueueRestartSentinelWake(message, sessionKey, wakeDeliveryContext);
|
||||
if (resolvedTo && channel) {
|
||||
const queuedNotice = await enqueueRestartSentinelNotice({
|
||||
channel,
|
||||
to: resolvedTo,
|
||||
accountId: origin?.accountId,
|
||||
replyToId,
|
||||
threadId: resolvedThreadId,
|
||||
message,
|
||||
sessionKey: canonicalKey,
|
||||
revision: sentinelRevision,
|
||||
});
|
||||
noticeQueueId = queuedNotice.id;
|
||||
noticeQueueCreated = queuedNotice.created;
|
||||
}
|
||||
|
||||
if (resolvedTo && channel) {
|
||||
const outboundSession = buildOutboundSessionContext({
|
||||
cfg,
|
||||
// Every downstream intent is durable before consuming the singleton. A
|
||||
// failed or stale compare-delete cannot lose work or remove a newer row.
|
||||
const consumed = await clearRestartSentinelIfRevision(sentinelRevision);
|
||||
if (!consumed) {
|
||||
log.info(`${summary}: newer restart sentinel preserved while draining durable work`, {
|
||||
sessionKey: canonicalKey,
|
||||
});
|
||||
}
|
||||
|
||||
if (wakeQueueId) {
|
||||
await drainRestartContinuationQueue({ deps: params.deps, entryId: wakeQueueId, log });
|
||||
}
|
||||
|
||||
if (resolvedTo && channel && noticeQueueId && noticeQueueCreated) {
|
||||
await deliverRestartSentinelNotice({
|
||||
deps: params.deps,
|
||||
cfg,
|
||||
@@ -668,7 +650,11 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
accountId: origin?.accountId,
|
||||
replyToId,
|
||||
threadId: resolvedThreadId,
|
||||
session: outboundSession,
|
||||
queueId: noticeQueueId,
|
||||
});
|
||||
} else if (noticeQueueId && !noticeQueueCreated) {
|
||||
log.info(`${summary}: durable restart notice already owned`, {
|
||||
sessionKey: canonicalKey,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -220,6 +220,45 @@ describe("delivery-queue-sqlite corrupt JSON resilience", () => {
|
||||
last_error: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("never prunes a permanent producer receipt", () => {
|
||||
upsertDeliveryQueueEntry({
|
||||
queueName: QUEUE,
|
||||
entry: {
|
||||
id: "rt-permanent",
|
||||
enqueuedAt: 1,
|
||||
retryCount: 0,
|
||||
completionRetention: "permanent",
|
||||
},
|
||||
stateDir,
|
||||
});
|
||||
completeDeliveryQueueEntry(QUEUE, "rt-permanent", stateDir);
|
||||
const { db } = openOpenClawStateDatabase({
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
db.prepare(
|
||||
`UPDATE delivery_queue_entries
|
||||
SET enqueued_at = ?
|
||||
WHERE queue_name = ? AND id = ?`,
|
||||
).run(Date.now() - 31 * 24 * 60 * 60_000, QUEUE, "rt-permanent");
|
||||
|
||||
enqueueValid("rt-prune-trigger");
|
||||
completeDeliveryQueueEntry(QUEUE, "rt-prune-trigger", stateDir);
|
||||
|
||||
expect(getDeliveryQueueEntryStatus(QUEUE, "rt-permanent", stateDir)).toBe("completed");
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT recovery_state, entry_json
|
||||
FROM delivery_queue_entries
|
||||
WHERE queue_name = ? AND id = ?`,
|
||||
)
|
||||
.get(QUEUE, "rt-permanent") as Record<string, unknown>;
|
||||
expect(row.recovery_state).toBe("completed_permanent");
|
||||
expect(JSON.parse(String(row.entry_json))).toMatchObject({
|
||||
completionRetention: "permanent",
|
||||
recoveryState: "completed_permanent",
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -13,6 +13,9 @@ import { runSqliteImmediateTransactionSync } from "./sqlite-transaction.js";
|
||||
type QueueStatus = "pending" | "failed" | "completed";
|
||||
type DeliveryQueueDatabase = Pick<OpenClawStateKyselyDatabase, "delivery_queue_entries">;
|
||||
const COMPLETED_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
||||
const PERMANENT_COMPLETION_RECOVERY_STATE = "completed_permanent";
|
||||
|
||||
export type DeliveryQueueCompletionRetention = "permanent";
|
||||
|
||||
/** Indexed metadata extracted from queue payloads for diagnostics and recovery. */
|
||||
export type DeliveryQueueRowMetadata = {
|
||||
@@ -28,6 +31,9 @@ export type DeliveryQueueEntryState = {
|
||||
id: string;
|
||||
enqueuedAt: number;
|
||||
retryCount: number;
|
||||
/** Durable delivery-call count reserved before invoking the provider path. */
|
||||
attemptCount?: number;
|
||||
completionRetention?: DeliveryQueueCompletionRetention;
|
||||
acknowledgedAt?: number;
|
||||
lastAttemptAt?: number;
|
||||
lastError?: string;
|
||||
@@ -433,11 +439,19 @@ export function deleteDeliveryQueueEntry(queueName: string, id: string, stateDir
|
||||
/** Retain a delivered row as a durable idempotency tombstone. */
|
||||
export function completeDeliveryQueueEntry(queueName: string, id: string, stateDir?: string): void {
|
||||
const now = Date.now();
|
||||
const current = loadDeliveryQueueEntry(queueName, id, stateDir);
|
||||
const retainPermanently = current?.completionRetention === "permanent";
|
||||
const tombstone = {
|
||||
id,
|
||||
enqueuedAt: now,
|
||||
retryCount: 0,
|
||||
acknowledgedAt: now,
|
||||
...(retainPermanently
|
||||
? {
|
||||
completionRetention: "permanent" as const,
|
||||
recoveryState: PERMANENT_COMPLETION_RECOVERY_STATE,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const completed = upsertDeliveryQueueEntry({
|
||||
queueName,
|
||||
@@ -453,7 +467,8 @@ export function completeDeliveryQueueEntry(queueName: string, id: string, stateD
|
||||
}
|
||||
throw enoent(queueName, id);
|
||||
}
|
||||
// Thirty days covers delayed producer replays while bounding successful-row growth.
|
||||
// Ordinary receipts expire after thirty days. Permanent producer receipts
|
||||
// survive because their source intent can outlive any bounded retry window.
|
||||
const database = openStateDatabase(stateDir);
|
||||
const queueDb = getNodeSqliteKysely<DeliveryQueueDatabase>(database.db);
|
||||
executeSqliteQuerySync(
|
||||
@@ -462,7 +477,13 @@ export function completeDeliveryQueueEntry(queueName: string, id: string, stateD
|
||||
.deleteFrom("delivery_queue_entries")
|
||||
.where("queue_name", "=", queueName)
|
||||
.where("status", "=", "completed")
|
||||
.where("enqueued_at", "<", now - COMPLETED_TOMBSTONE_RETENTION_MS),
|
||||
.where("enqueued_at", "<", now - COMPLETED_TOMBSTONE_RETENTION_MS)
|
||||
.where((eb) =>
|
||||
eb.or([
|
||||
eb("recovery_state", "is", null),
|
||||
eb("recovery_state", "!=", PERMANENT_COMPLETION_RECOVERY_STATE),
|
||||
]),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -480,6 +501,59 @@ export function updateDeliveryQueueEntry(
|
||||
upsertDeliveryQueueEntry({ queueName, entry: update(current), stateDir });
|
||||
}
|
||||
|
||||
type ReserveDeliveryQueueAttemptResult =
|
||||
| { status: "reserved"; attemptCount: number }
|
||||
| { status: "exhausted"; attemptCount: number };
|
||||
|
||||
/** Atomically reserve one provider-delivery call before executing it. */
|
||||
export function reserveDeliveryQueueEntryAttempt(params: {
|
||||
queueName: string;
|
||||
id: string;
|
||||
maxAttempts: number;
|
||||
stateDir?: string;
|
||||
}): ReserveDeliveryQueueAttemptResult {
|
||||
if (!Number.isInteger(params.maxAttempts) || params.maxAttempts <= 0) {
|
||||
throw new Error(`Invalid delivery attempt budget: ${params.maxAttempts}`);
|
||||
}
|
||||
const database = openStateDatabase(params.stateDir);
|
||||
return runSqliteImmediateTransactionSync(
|
||||
database.db,
|
||||
() => {
|
||||
const current = loadDeliveryQueueEntry(params.queueName, params.id, params.stateDir);
|
||||
if (!current) {
|
||||
throw enoent(params.queueName, params.id);
|
||||
}
|
||||
const persistedAttemptCount =
|
||||
typeof current.attemptCount === "number" &&
|
||||
Number.isInteger(current.attemptCount) &&
|
||||
current.attemptCount >= 0
|
||||
? current.attemptCount
|
||||
: 0;
|
||||
const attemptCount = Math.max(persistedAttemptCount, current.retryCount);
|
||||
if (attemptCount >= params.maxAttempts) {
|
||||
return { status: "exhausted", attemptCount };
|
||||
}
|
||||
const reservedAttemptCount = attemptCount + 1;
|
||||
const updated = upsertDeliveryQueueEntryInDatabase(
|
||||
{
|
||||
queueName: params.queueName,
|
||||
entry: { ...current, attemptCount: reservedAttemptCount },
|
||||
updatePendingOnly: true,
|
||||
},
|
||||
database,
|
||||
);
|
||||
if (!updated) {
|
||||
throw enoent(params.queueName, params.id);
|
||||
}
|
||||
return { status: "reserved", attemptCount: reservedAttemptCount };
|
||||
},
|
||||
{
|
||||
databaseLabel: "openclaw-state",
|
||||
operationLabel: `reserve ${params.queueName} delivery attempt`,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Dead-lettered entry counts for one queue namespace. */
|
||||
type FailedDeliveryQueueCount = {
|
||||
queueName: string;
|
||||
|
||||
@@ -52,6 +52,7 @@ import {
|
||||
loadPendingDeliveries,
|
||||
markDeliveryPlatformOutcomeUnknown,
|
||||
moveToFailed,
|
||||
reserveDeliveryAttempt,
|
||||
type QueuedDelivery,
|
||||
type QueuedDeliveryPayload,
|
||||
} from "./delivery-queue-storage.js";
|
||||
@@ -98,7 +99,7 @@ type ActiveDeliveryClaimResult<T> =
|
||||
| { status: "claimed"; value: T }
|
||||
| { status: "claimed-by-other-owner" };
|
||||
|
||||
const MAX_RETRIES = 5;
|
||||
const DEFAULT_MAX_RETRIES = 5;
|
||||
|
||||
const PERMANENT_ERROR_PATTERNS: readonly RegExp[] = [
|
||||
/no conversation reference found/i,
|
||||
@@ -118,6 +119,20 @@ const drainInProgress = new Map<string, boolean>();
|
||||
const entriesInProgress = new Set<string>();
|
||||
const recoveryReplayPacer = createRecoveryReplayPacer();
|
||||
|
||||
function resolveMaxRetries(entry: QueuedDelivery): number {
|
||||
const configured = entry.maxRetries;
|
||||
return typeof configured === "number" && Number.isInteger(configured) && configured > 0
|
||||
? configured
|
||||
: DEFAULT_MAX_RETRIES;
|
||||
}
|
||||
|
||||
function resolveAttemptCount(entry: QueuedDelivery): number {
|
||||
const persisted = entry.attemptCount;
|
||||
const attemptCount =
|
||||
typeof persisted === "number" && Number.isInteger(persisted) && persisted >= 0 ? persisted : 0;
|
||||
return Math.max(attemptCount, entry.retryCount);
|
||||
}
|
||||
|
||||
function resolveRecoveryDeadlineMs(maxRecoveryMs: number | undefined): number {
|
||||
const durationMs =
|
||||
typeof maxRecoveryMs === "number" && Number.isFinite(maxRecoveryMs)
|
||||
@@ -150,10 +165,14 @@ function emitQueuedAuditTerminals(
|
||||
});
|
||||
}
|
||||
|
||||
function needsUnknownSendReconciliation(entry: QueuedDelivery): boolean {
|
||||
return (
|
||||
entry.recoveryState === "send_attempt_started" || entry.recoveryState === "unknown_after_send"
|
||||
);
|
||||
}
|
||||
|
||||
function queuedDeadLetterAuditTerminals(entry: QueuedDelivery) {
|
||||
const ambiguous =
|
||||
entry.recoveryState === "send_attempt_started" || entry.recoveryState === "unknown_after_send";
|
||||
if (ambiguous) {
|
||||
if (needsUnknownSendReconciliation(entry)) {
|
||||
return uniformOutboundAuditTerminals(entry.payloads.length, {
|
||||
outcome: "unknown",
|
||||
failureStage: "queue",
|
||||
@@ -597,14 +616,13 @@ async function drainQueuedEntry(opts: {
|
||||
onFailed?: (entry: QueuedDelivery, errMsg: string) => void;
|
||||
}): Promise<"recovered" | "failed" | "moved-to-failed" | "already-gone"> {
|
||||
const { entry } = opts;
|
||||
const maxRetries = resolveMaxRetries(entry);
|
||||
const attemptBudgetExhausted = resolveAttemptCount(entry) >= maxRetries;
|
||||
const ownerState = await resolveCompletedOwnerBeforeRecovery(opts);
|
||||
if (ownerState !== "continue") {
|
||||
return ownerState;
|
||||
}
|
||||
if (
|
||||
entry.recoveryState === "send_attempt_started" ||
|
||||
entry.recoveryState === "unknown_after_send"
|
||||
) {
|
||||
if (needsUnknownSendReconciliation(entry)) {
|
||||
// A crash after platform send start cannot be blindly replayed; adapters
|
||||
// must reconcile whether the platform already committed the message.
|
||||
const reconciliation = await reconcileUnknownQueuedDelivery({
|
||||
@@ -668,7 +686,11 @@ async function drainQueuedEntry(opts: {
|
||||
}
|
||||
opts.log.warn(`Delivery entry ${entry.id} ${errMsg}`);
|
||||
opts.onFailed?.(entry, errMsg);
|
||||
if (reconciliation?.status === "unresolved" && reconciliation.retryable === true) {
|
||||
if (
|
||||
reconciliation?.status === "unresolved" &&
|
||||
reconciliation.retryable === true &&
|
||||
!attemptBudgetExhausted
|
||||
) {
|
||||
try {
|
||||
await failDelivery(entry.id, errMsg, opts.stateDir);
|
||||
return "failed";
|
||||
@@ -715,6 +737,22 @@ async function drainQueuedEntry(opts: {
|
||||
commitHooksRun = true;
|
||||
await runOutboundDeliveryCommitHooks(deliveredResults);
|
||||
};
|
||||
const reservation = await reserveDeliveryAttempt(entry.id, maxRetries, opts.stateDir);
|
||||
if (reservation.status === "exhausted") {
|
||||
const errMsg = `delivery retry budget exhausted (${reservation.attemptCount}/${maxRetries})`;
|
||||
markDurableDeliveryFailedBestEffort(entry, opts.log);
|
||||
try {
|
||||
await moveToFailed(entry.id, opts.stateDir);
|
||||
} catch (moveErr) {
|
||||
if (getErrnoCode(moveErr) === "ENOENT") {
|
||||
return "already-gone";
|
||||
}
|
||||
throw moveErr;
|
||||
}
|
||||
emitQueuedAuditTerminals(entry, () => queuedDeadLetterAuditTerminals(entry));
|
||||
opts.onFailed?.(entry, errMsg);
|
||||
return "moved-to-failed";
|
||||
}
|
||||
const recoverySpoolPaths = collectEntrySpoolPaths(entry.payloads, opts.stateDir);
|
||||
let mediaRecoveryLeaseId: string | undefined;
|
||||
try {
|
||||
@@ -980,7 +1018,11 @@ export async function drainPendingDeliveries(opts: {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (currentEntry.retryCount >= MAX_RETRIES) {
|
||||
const maxRetries = resolveMaxRetries(currentEntry);
|
||||
if (
|
||||
resolveAttemptCount(currentEntry) >= maxRetries &&
|
||||
!needsUnknownSendReconciliation(currentEntry)
|
||||
) {
|
||||
try {
|
||||
markDurableDeliveryFailedBestEffort(currentEntry, opts.log);
|
||||
await moveToFailed(currentEntry.id, opts.stateDir);
|
||||
@@ -1044,7 +1086,7 @@ export async function drainPendingDeliveries(opts: {
|
||||
|
||||
/**
|
||||
* On gateway startup, scan the delivery queue and retry any pending entries.
|
||||
* Uses exponential backoff and moves entries that exceed MAX_RETRIES to failed/.
|
||||
* Uses exponential backoff and moves entries that exhaust their retry budget to failed/.
|
||||
*/
|
||||
export async function recoverPendingDeliveries(opts: {
|
||||
deliver: DeliverFn;
|
||||
@@ -1100,9 +1142,11 @@ export async function recoverPendingDeliveries(opts: {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (currentEntry.retryCount >= MAX_RETRIES) {
|
||||
const maxRetries = resolveMaxRetries(currentEntry);
|
||||
const attemptCount = resolveAttemptCount(currentEntry);
|
||||
if (attemptCount >= maxRetries && !needsUnknownSendReconciliation(currentEntry)) {
|
||||
opts.log.warn(
|
||||
`Delivery ${currentEntry.id} exceeded max retries (${currentEntry.retryCount}/${MAX_RETRIES}) — moving to failed/`,
|
||||
`Delivery ${currentEntry.id} exceeded max retries (${attemptCount}/${maxRetries}) — moving to failed/`,
|
||||
);
|
||||
const movedToFailed = await moveEntryToFailedWithLogging(
|
||||
currentEntry,
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { RenderedMessageBatchPlanItem } from "../../channels/message/types.
|
||||
import type { ReplyToMode } from "../../config/types.js";
|
||||
import type { PluginHookReplyPayloadSendingContext } from "../../plugins/hook-types.js";
|
||||
import {
|
||||
completeDeliveryQueueEntry,
|
||||
commitStagedDeliveryQueueEntry,
|
||||
commitStagedDeliveryQueueEntryOnce,
|
||||
deleteDeliveryQueueEntry,
|
||||
@@ -13,9 +14,11 @@ import {
|
||||
loadDeliveryQueueEntries,
|
||||
loadDeliveryQueueEntry,
|
||||
moveDeliveryQueueEntryToFailed,
|
||||
reserveDeliveryQueueEntryAttempt,
|
||||
updateDeliveryQueueEntry,
|
||||
upsertDeliveryQueueEntry,
|
||||
type DeliveryQueueRowMetadata,
|
||||
type DeliveryQueueCompletionRetention,
|
||||
} from "../delivery-queue-sqlite.js";
|
||||
import { generateSecureUuid } from "../secure-random.js";
|
||||
import type { DurableDeliveryCompletion } from "./delivery-completion.js";
|
||||
@@ -85,12 +88,17 @@ export type QueuedDeliveryPayload = {
|
||||
preparedMessageId?: string;
|
||||
/** Serializable owner state finalized by both live delivery and recovery. */
|
||||
deliveryCompletion?: DurableDeliveryCompletion;
|
||||
/** Retain a terminal receipt when the producer may replay this stable intent indefinitely. */
|
||||
completionRetention?: DeliveryQueueCompletionRetention;
|
||||
/** Producer-specific retry budget; omitted entries use the queue default. */
|
||||
maxRetries?: number;
|
||||
};
|
||||
|
||||
export interface QueuedDelivery extends QueuedDeliveryPayload {
|
||||
id: string;
|
||||
enqueuedAt: number;
|
||||
retryCount: number;
|
||||
attemptCount: number;
|
||||
lastAttemptAt?: number;
|
||||
lastError?: string;
|
||||
platformSendStartedAt?: number;
|
||||
@@ -135,7 +143,10 @@ function createQueuedDelivery(params: QueuedDeliveryPayload, id: string): Queued
|
||||
gatewayClientScopes: params.gatewayClientScopes,
|
||||
preparedMessageId: params.preparedMessageId,
|
||||
deliveryCompletion: params.deliveryCompletion,
|
||||
completionRetention: params.completionRetention,
|
||||
maxRetries: params.maxRetries,
|
||||
retryCount: 0,
|
||||
attemptCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -224,7 +235,7 @@ type AckDeliveryOptions = {
|
||||
retainSpoolArtifacts?: boolean;
|
||||
};
|
||||
|
||||
/** Remove a successfully delivered entry from the queue. */
|
||||
/** Remove a successfully delivered entry, or retain its permanent producer receipt. */
|
||||
export async function ackDelivery(
|
||||
id: string,
|
||||
stateDir?: string,
|
||||
@@ -233,8 +244,17 @@ export async function ackDelivery(
|
||||
// Read the media references before the row goes, then unlink only after the
|
||||
// delete commits. A crash in between leaves an orphan for the retention sweep;
|
||||
// unlinking first could strip media from a row that still has to replay.
|
||||
const spoolPaths = loadEntrySpoolPaths(id, stateDir);
|
||||
deleteDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir);
|
||||
const entry = loadDeliveryQueueEntry(
|
||||
OUTBOUND_DELIVERY_QUEUE_NAME,
|
||||
id,
|
||||
stateDir,
|
||||
) as QueuedDelivery | null;
|
||||
const spoolPaths = entry ? collectEntrySpoolPaths(entry.payloads, stateDir) : [];
|
||||
if (entry?.completionRetention === "permanent") {
|
||||
completeDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir);
|
||||
} else {
|
||||
deleteDeliveryQueueEntry(OUTBOUND_DELIVERY_QUEUE_NAME, id, stateDir);
|
||||
}
|
||||
if (!options?.retainSpoolArtifacts) {
|
||||
await releaseSpoolArtifacts(spoolPaths, stateDir);
|
||||
}
|
||||
@@ -283,6 +303,16 @@ export async function failDeliveryAfterPlatformSend(
|
||||
}));
|
||||
}
|
||||
|
||||
/** Reserve one durable delivery call before invoking the provider path. */
|
||||
export async function reserveDeliveryAttempt(id: string, maxAttempts: number, stateDir?: string) {
|
||||
return reserveDeliveryQueueEntryAttempt({
|
||||
queueName: OUTBOUND_DELIVERY_QUEUE_NAME,
|
||||
id,
|
||||
maxAttempts,
|
||||
stateDir,
|
||||
});
|
||||
}
|
||||
|
||||
function updateQueuedDelivery(
|
||||
id: string,
|
||||
stateDir: string | undefined,
|
||||
|
||||
@@ -4,13 +4,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { controlNextRecoverySleep } from "../../../test/helpers/infra/delivery-recovery.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { openOpenClawStateDatabase } from "../../state/openclaw-state-db.js";
|
||||
import { loadPendingDeliveries } from "./delivery-queue-storage.js";
|
||||
import { loadPendingDeliveries, reserveDeliveryAttempt } from "./delivery-queue-storage.js";
|
||||
import {
|
||||
type DeliverFn,
|
||||
drainPendingDeliveries,
|
||||
enqueueDelivery,
|
||||
failDelivery,
|
||||
markDeliveryPlatformOutcomeUnknown,
|
||||
markDeliveryPlatformSendAttemptStarted,
|
||||
type RecoveryLogger,
|
||||
recoverPendingDeliveries,
|
||||
withActiveDeliveryClaim,
|
||||
@@ -27,8 +28,12 @@ const MAX_RETRIES = 5;
|
||||
const stubCfg = {} as OpenClawConfig;
|
||||
const NO_LISTENER_ERROR = "No active DirectChat listener";
|
||||
const sleepMock = vi.hoisted(() => vi.fn<(ms: number) => Promise<void>>());
|
||||
const resolveOutboundChannelMessageAdapterMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../../utils/sleep.js", () => ({ sleep: sleepMock }));
|
||||
vi.mock("./channel-resolution.js", () => ({
|
||||
resolveOutboundChannelMessageAdapter: resolveOutboundChannelMessageAdapterMock,
|
||||
}));
|
||||
|
||||
function normalizeReconnectAccountIdForTest(accountId?: string | null): string {
|
||||
return (accountId ?? "").trim() || "default";
|
||||
@@ -146,6 +151,7 @@ describe("drainPendingDeliveries for reconnect", () => {
|
||||
tmpDir = fixtures.tmpDir();
|
||||
sleepMock.mockReset();
|
||||
sleepMock.mockResolvedValue(undefined);
|
||||
resolveOutboundChannelMessageAdapterMock.mockReset();
|
||||
});
|
||||
|
||||
it("drains entries that failed with 'no listener' error", async () => {
|
||||
@@ -218,6 +224,46 @@ describe("drainPendingDeliveries for reconnect", () => {
|
||||
expectLogMessageWith(log.warn, "refusing blind replay without adapter reconciliation");
|
||||
});
|
||||
|
||||
it("reconciles an exhausted final attempt before dead-lettering", async () => {
|
||||
const log = createRecoveryLog();
|
||||
const deliver = vi.fn<DeliverFn>(async () => {});
|
||||
const id = await enqueueDelivery(
|
||||
{
|
||||
channel: "directchat",
|
||||
to: "+1555",
|
||||
payloads: [{ text: "maybe sent" }],
|
||||
accountId: "acct1",
|
||||
maxRetries: 1,
|
||||
},
|
||||
tmpDir,
|
||||
);
|
||||
await reserveDeliveryAttempt(id, 1, tmpDir);
|
||||
await markDeliveryPlatformSendAttemptStarted(id, tmpDir);
|
||||
const reconcileUnknownSend = vi.fn().mockResolvedValue({
|
||||
status: "sent",
|
||||
messageId: "platform-final",
|
||||
receipt: {
|
||||
primaryPlatformMessageId: "platform-final",
|
||||
platformMessageIds: ["platform-final"],
|
||||
parts: [{ platformMessageId: "platform-final", kind: "text", index: 0 }],
|
||||
sentAt: 1,
|
||||
},
|
||||
});
|
||||
resolveOutboundChannelMessageAdapterMock.mockReturnValue({
|
||||
durableFinal: {
|
||||
capabilities: { reconcileUnknownSend: true },
|
||||
reconcileUnknownSend,
|
||||
},
|
||||
});
|
||||
|
||||
await drainAcct1DirectChatReconnect({ deliver, log, stateDir: tmpDir });
|
||||
|
||||
expect(deliver).not.toHaveBeenCalled();
|
||||
expect(reconcileUnknownSend).toHaveBeenCalledOnce();
|
||||
expect(await loadPendingDeliveries(tmpDir)).toHaveLength(0);
|
||||
expect(readOutboundQueueStatus(tmpDir, id)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("skips entries where retryCount >= MAX_RETRIES", async () => {
|
||||
const log = createRecoveryLog();
|
||||
const deliver = vi.fn<DeliverFn>(async () => {});
|
||||
|
||||
@@ -24,7 +24,7 @@ import {
|
||||
} from "./deliver-types.js";
|
||||
import { attachOutboundDeliveryCommitHook } from "./delivery-commit-hooks.js";
|
||||
import { pruneOrphanedDeliveryQueueMedia } from "./delivery-queue-media-spool.js";
|
||||
import { loadPendingDeliveries } from "./delivery-queue-storage.js";
|
||||
import { loadPendingDeliveries, reserveDeliveryAttempt } from "./delivery-queue-storage.js";
|
||||
import {
|
||||
ackDelivery,
|
||||
enqueueDelivery,
|
||||
@@ -498,6 +498,67 @@ describe("delivery-queue recovery", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("honors a producer-specific retry budget", async () => {
|
||||
const id = await enqueueDelivery(
|
||||
{
|
||||
channel: "demo-channel-a",
|
||||
to: "+1",
|
||||
payloads: [{ text: "a" }],
|
||||
maxRetries: 45,
|
||||
},
|
||||
tmpDir(),
|
||||
);
|
||||
setQueuedEntryState(tmpDir(), id, {
|
||||
retryCount: MAX_RETRIES,
|
||||
lastAttemptAt: Date.now() - 10_000_000,
|
||||
});
|
||||
const deliver = vi.fn().mockResolvedValue([]);
|
||||
|
||||
const { result } = await runRecovery({ deliver });
|
||||
|
||||
expect(deliver).toHaveBeenCalledOnce();
|
||||
expect(result).toMatchObject({ recovered: 1, skippedMaxRetries: 0 });
|
||||
expect(readOutboundQueueStatus(tmpDir(), id)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("dead-letters an atomically exhausted attempt budget before replay", async () => {
|
||||
const id = await enqueueDelivery(
|
||||
{
|
||||
channel: "demo-channel-a",
|
||||
to: "+1",
|
||||
payloads: [{ text: "a" }],
|
||||
maxRetries: 1,
|
||||
},
|
||||
tmpDir(),
|
||||
);
|
||||
await reserveDeliveryAttempt(id, 1, tmpDir());
|
||||
const deliver = vi.fn();
|
||||
|
||||
const { result } = await runRecovery({ deliver });
|
||||
|
||||
expect(deliver).not.toHaveBeenCalled();
|
||||
expect(result.skippedMaxRetries).toBe(1);
|
||||
expect(readOutboundQueueStatus(tmpDir(), id)).toBe("failed");
|
||||
});
|
||||
|
||||
it("ignores an invalid producer retry budget", async () => {
|
||||
await enqueueDelivery(
|
||||
{
|
||||
channel: "demo-channel-a",
|
||||
to: "+1",
|
||||
payloads: [{ text: "a" }],
|
||||
maxRetries: 0.5,
|
||||
},
|
||||
tmpDir(),
|
||||
);
|
||||
const deliver = vi.fn().mockResolvedValue([]);
|
||||
|
||||
const { result } = await runRecovery({ deliver });
|
||||
|
||||
expect(deliver).toHaveBeenCalledOnce();
|
||||
expect(result).toMatchObject({ recovered: 1, skippedMaxRetries: 0 });
|
||||
});
|
||||
|
||||
it("dead-letters max-retry entries even when conversation owner state is missing", async () => {
|
||||
const storePath = path.join(tmpDir(), "missing-owner-sessions.json");
|
||||
const id = await enqueueDelivery(
|
||||
@@ -537,6 +598,7 @@ describe("delivery-queue recovery", () => {
|
||||
);
|
||||
setQueuedEntryState(tmpDir(), id, {
|
||||
retryCount: MAX_RETRIES,
|
||||
lastAttemptAt: Date.now() - 10_000_000,
|
||||
platformSendStartedAt: Date.now(),
|
||||
recoveryState: "send_attempt_started",
|
||||
});
|
||||
@@ -544,7 +606,7 @@ describe("delivery-queue recovery", () => {
|
||||
const { result } = await runRecovery({ deliver: vi.fn() });
|
||||
unsubscribe();
|
||||
|
||||
expect(result.skippedMaxRetries).toBe(1);
|
||||
expect(result).toMatchObject({ failed: 1, skippedMaxRetries: 0 });
|
||||
expect(auditEvents).toHaveLength(1);
|
||||
expect(auditEvents[0]).toMatchObject({
|
||||
sourceId: `message:outbound:queue:${id}:payload:0`,
|
||||
@@ -571,6 +633,7 @@ describe("delivery-queue recovery", () => {
|
||||
const entries = await loadPendingDeliveries(tmpDir());
|
||||
expect(entries).toHaveLength(1);
|
||||
expect(entries[0]?.retryCount).toBe(1);
|
||||
expect(entries[0]?.attemptCount).toBe(1);
|
||||
expect(entries[0]?.lastError).toBe("network down");
|
||||
expect(auditEvents).toEqual([]);
|
||||
});
|
||||
@@ -1008,9 +1071,11 @@ describe("delivery-queue recovery", () => {
|
||||
replyToId: "root-message",
|
||||
threadId: "thread-1",
|
||||
silent: true,
|
||||
maxRetries: 1,
|
||||
},
|
||||
tmpDir(),
|
||||
);
|
||||
await reserveDeliveryAttempt(id, 1, tmpDir());
|
||||
await markDeliveryPlatformSendAttemptStarted(id, tmpDir(), {
|
||||
replyToId: "hooked-root-message",
|
||||
});
|
||||
@@ -1172,6 +1237,45 @@ describe("delivery-queue recovery", () => {
|
||||
expect(entries[0]?.lastError).toContain("provider lookup timed out");
|
||||
});
|
||||
|
||||
it("dead-letters an exhausted unknown send after retryable reconciliation fails", async () => {
|
||||
const id = await enqueueDelivery(
|
||||
{
|
||||
channel: "demo-channel-a",
|
||||
to: "+1",
|
||||
payloads: [{ text: "unknown final attempt" }],
|
||||
maxRetries: 1,
|
||||
},
|
||||
tmpDir(),
|
||||
);
|
||||
await reserveDeliveryAttempt(id, 1, tmpDir());
|
||||
setQueuedEntryState(tmpDir(), id, {
|
||||
retryCount: 0,
|
||||
lastAttemptAt: Date.now() - 10_000_000,
|
||||
platformSendStartedAt: Date.now(),
|
||||
recoveryState: "unknown_after_send",
|
||||
});
|
||||
const reconcileUnknownSend = vi.fn().mockResolvedValue({
|
||||
status: "unresolved",
|
||||
error: "provider lookup timed out",
|
||||
retryable: true,
|
||||
});
|
||||
resolveOutboundChannelMessageAdapterMock.mockReturnValue({
|
||||
durableFinal: {
|
||||
capabilities: { reconcileUnknownSend: true },
|
||||
reconcileUnknownSend,
|
||||
},
|
||||
});
|
||||
const deliver = vi.fn().mockResolvedValue([]);
|
||||
|
||||
const { result } = await runRecovery({ deliver });
|
||||
|
||||
expect(reconcileUnknownSend).toHaveBeenCalledOnce();
|
||||
expect(deliver).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({ failed: 1, skippedMaxRetries: 0 });
|
||||
expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0);
|
||||
expect(readOutboundQueueStatus(tmpDir(), id)).toBe("failed");
|
||||
});
|
||||
|
||||
it("does not reconcile unknown-after-send entries unless the adapter declares the capability", async () => {
|
||||
const id = await enqueueDelivery(
|
||||
{ channel: "demo-channel-a", to: "+1", payloads: [{ text: "hidden method" }] },
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
loadPendingDelivery,
|
||||
loadPendingDeliveries,
|
||||
moveToFailed,
|
||||
reserveDeliveryAttempt,
|
||||
} from "./delivery-queue-storage.js";
|
||||
import {
|
||||
ackDelivery,
|
||||
@@ -55,6 +56,40 @@ describe("delivery-queue storage", () => {
|
||||
}
|
||||
|
||||
describe("enqueue + ack lifecycle", () => {
|
||||
it("persists a producer-specific retry budget", async () => {
|
||||
const id = await enqueueTextDelivery({
|
||||
channel: "directchat",
|
||||
to: "+1555",
|
||||
payloads: [{ text: "retry-budget" }],
|
||||
maxRetries: 45,
|
||||
});
|
||||
|
||||
expect(readQueuedEntry(tmpDir(), id).maxRetries).toBe(45);
|
||||
});
|
||||
|
||||
it("atomically reserves delivery attempts up to the producer budget", async () => {
|
||||
const id = await enqueueTextDelivery({
|
||||
channel: "directchat",
|
||||
to: "+1555",
|
||||
payloads: [{ text: "attempt-budget" }],
|
||||
maxRetries: 2,
|
||||
});
|
||||
|
||||
await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({
|
||||
status: "reserved",
|
||||
attemptCount: 1,
|
||||
});
|
||||
await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({
|
||||
status: "reserved",
|
||||
attemptCount: 2,
|
||||
});
|
||||
await expect(reserveDeliveryAttempt(id, 2, tmpDir())).resolves.toEqual({
|
||||
status: "exhausted",
|
||||
attemptCount: 2,
|
||||
});
|
||||
expect(readQueuedEntry(tmpDir(), id).attemptCount).toBe(2);
|
||||
});
|
||||
|
||||
it("creates and removes a queue entry", async () => {
|
||||
const id = await enqueueTextDelivery(
|
||||
{
|
||||
@@ -180,6 +215,36 @@ describe("delivery-queue storage", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps permanent completion ownership after ack", async () => {
|
||||
const id = "restart-sentinel-notice:agent:main:main:123";
|
||||
await enqueueDeliveryOnce(
|
||||
{
|
||||
channel: "directchat",
|
||||
to: "+1555",
|
||||
payloads: [{ text: "restart complete" }],
|
||||
completionRetention: "permanent",
|
||||
},
|
||||
id,
|
||||
tmpDir(),
|
||||
);
|
||||
|
||||
await ackDelivery(id, tmpDir());
|
||||
const repeated = await enqueueDeliveryOnce(
|
||||
{
|
||||
channel: "directchat",
|
||||
to: "+1555",
|
||||
payloads: [{ text: "must not replay" }],
|
||||
completionRetention: "permanent",
|
||||
},
|
||||
id,
|
||||
tmpDir(),
|
||||
);
|
||||
|
||||
expect(repeated).toEqual({ id, created: false });
|
||||
expect(await loadPendingDeliveries(tmpDir())).toEqual([]);
|
||||
expect(readStatus(id)).toBe("completed");
|
||||
});
|
||||
|
||||
it("ack is idempotent (no error on missing file)", async () => {
|
||||
await expect(ackDelivery("nonexistent-id", tmpDir())).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,660 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "./kysely-sync.js";
|
||||
|
||||
type RestartSentinelLog = {
|
||||
stdoutTail?: string | null;
|
||||
stderrTail?: string | null;
|
||||
exitCode?: number | null;
|
||||
};
|
||||
|
||||
type RestartSentinelStep = {
|
||||
name: string;
|
||||
command: string;
|
||||
cwd?: string | null;
|
||||
durationMs?: number | null;
|
||||
log?: RestartSentinelLog | null;
|
||||
};
|
||||
|
||||
type RestartSentinelStats = {
|
||||
mode?: string;
|
||||
root?: string;
|
||||
requiresRestart?: boolean;
|
||||
handoffId?: string;
|
||||
before?: Record<string, unknown> | null;
|
||||
after?: Record<string, unknown> | null;
|
||||
steps?: RestartSentinelStep[];
|
||||
reason?: string | null;
|
||||
durationMs?: number | null;
|
||||
};
|
||||
|
||||
export type RestartSentinelContinuation =
|
||||
| {
|
||||
kind: "systemEvent";
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
kind: "agentTurn";
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type RestartSentinelPayload = {
|
||||
kind: "config-apply" | "config-auto-recovery" | "config-patch" | "update" | "restart";
|
||||
status: "ok" | "error" | "skipped";
|
||||
ts: number;
|
||||
sessionKey?: string;
|
||||
deliveryContext?: {
|
||||
channel?: string;
|
||||
to?: string;
|
||||
accountId?: string;
|
||||
};
|
||||
threadId?: string;
|
||||
message?: string | null;
|
||||
continuation?: RestartSentinelContinuation | null;
|
||||
doctorHint?: string | null;
|
||||
stats?: RestartSentinelStats | null;
|
||||
};
|
||||
|
||||
export type RestartSentinelEnvelope = {
|
||||
version: 1;
|
||||
payload: RestartSentinelPayload;
|
||||
};
|
||||
|
||||
export type RestartSentinel = RestartSentinelEnvelope & {
|
||||
/** Optimistic-concurrency revision backed by gateway_restart_sentinel.updated_at_ms. */
|
||||
revision: number;
|
||||
};
|
||||
|
||||
type RestartSentinelRowState =
|
||||
| { kind: "missing" }
|
||||
| { kind: "invalid"; revision: number }
|
||||
| { kind: "valid"; sentinel: RestartSentinel };
|
||||
|
||||
const RESTART_SENTINEL_KEY = "current";
|
||||
const RESTART_SENTINEL_REVISION_FLOOR_KEY = "revision-floor";
|
||||
const RESTART_SENTINEL_KINDS = new Set<RestartSentinelPayload["kind"]>([
|
||||
"config-apply",
|
||||
"config-auto-recovery",
|
||||
"config-patch",
|
||||
"update",
|
||||
"restart",
|
||||
]);
|
||||
const RESTART_SENTINEL_STATUSES = new Set<RestartSentinelPayload["status"]>([
|
||||
"ok",
|
||||
"error",
|
||||
"skipped",
|
||||
]);
|
||||
|
||||
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
|
||||
|
||||
function isFiniteNumber(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
function isSafeInteger(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isSafeInteger(value);
|
||||
}
|
||||
|
||||
function parseOptionalNullableString(
|
||||
record: Record<string, unknown>,
|
||||
key: string,
|
||||
): string | null | undefined | false {
|
||||
const value = record[key];
|
||||
if (value === undefined || value === null || typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function parseRestartSentinelLog(value: unknown): RestartSentinelLog | null {
|
||||
if (!isPlainRecord(value)) {
|
||||
return null;
|
||||
}
|
||||
const stdoutTail = parseOptionalNullableString(value, "stdoutTail");
|
||||
const stderrTail = parseOptionalNullableString(value, "stderrTail");
|
||||
const exitCode = value.exitCode;
|
||||
if (
|
||||
stdoutTail === false ||
|
||||
stderrTail === false ||
|
||||
(exitCode !== undefined && exitCode !== null && !isSafeInteger(exitCode))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const result: RestartSentinelLog = {};
|
||||
if (stdoutTail !== undefined) {
|
||||
result.stdoutTail = stdoutTail;
|
||||
}
|
||||
if (stderrTail !== undefined) {
|
||||
result.stderrTail = stderrTail;
|
||||
}
|
||||
if (exitCode !== undefined) {
|
||||
result.exitCode = exitCode as number | null;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseRestartSentinelStep(value: unknown): RestartSentinelStep | null {
|
||||
if (
|
||||
!isPlainRecord(value) ||
|
||||
typeof value.name !== "string" ||
|
||||
typeof value.command !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const cwd = parseOptionalNullableString(value, "cwd");
|
||||
const durationMs = value.durationMs;
|
||||
const log = value.log;
|
||||
if (
|
||||
cwd === false ||
|
||||
(durationMs !== undefined && durationMs !== null && !isFiniteNumber(durationMs)) ||
|
||||
(log !== undefined && log !== null && !parseRestartSentinelLog(log))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const result: RestartSentinelStep = { name: value.name, command: value.command };
|
||||
if (cwd !== undefined) {
|
||||
result.cwd = cwd;
|
||||
}
|
||||
if (durationMs !== undefined) {
|
||||
result.durationMs = durationMs as number | null;
|
||||
}
|
||||
if (log !== undefined) {
|
||||
result.log = log === null ? null : parseRestartSentinelLog(log);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseRestartSentinelStats(value: unknown): RestartSentinelStats | null {
|
||||
if (!isPlainRecord(value)) {
|
||||
return null;
|
||||
}
|
||||
const mode = parseOptionalNullableString(value, "mode");
|
||||
const root = parseOptionalNullableString(value, "root");
|
||||
const handoffId = parseOptionalNullableString(value, "handoffId");
|
||||
const reason = parseOptionalNullableString(value, "reason");
|
||||
const before = value.before;
|
||||
const after = value.after;
|
||||
const steps = value.steps;
|
||||
const durationMs = value.durationMs;
|
||||
if (
|
||||
mode === false ||
|
||||
mode === null ||
|
||||
root === false ||
|
||||
root === null ||
|
||||
handoffId === false ||
|
||||
handoffId === null ||
|
||||
reason === false ||
|
||||
(value.requiresRestart !== undefined && typeof value.requiresRestart !== "boolean") ||
|
||||
(before !== undefined && before !== null && !isPlainRecord(before)) ||
|
||||
(after !== undefined && after !== null && !isPlainRecord(after)) ||
|
||||
(steps !== undefined &&
|
||||
(!Array.isArray(steps) || steps.some((step) => !parseRestartSentinelStep(step)))) ||
|
||||
(durationMs !== undefined && durationMs !== null && !isFiniteNumber(durationMs))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const result: RestartSentinelStats = {};
|
||||
if (mode !== undefined) {
|
||||
result.mode = mode;
|
||||
}
|
||||
if (root !== undefined) {
|
||||
result.root = root;
|
||||
}
|
||||
if (value.requiresRestart !== undefined) {
|
||||
result.requiresRestart = value.requiresRestart as boolean;
|
||||
}
|
||||
if (handoffId !== undefined) {
|
||||
result.handoffId = handoffId;
|
||||
}
|
||||
if (before !== undefined) {
|
||||
result.before = before as Record<string, unknown> | null;
|
||||
}
|
||||
if (after !== undefined) {
|
||||
result.after = after as Record<string, unknown> | null;
|
||||
}
|
||||
if (steps !== undefined) {
|
||||
result.steps = steps.map((step) => parseRestartSentinelStep(step)!);
|
||||
}
|
||||
if (reason !== undefined) {
|
||||
result.reason = reason;
|
||||
}
|
||||
if (durationMs !== undefined) {
|
||||
result.durationMs = durationMs as number | null;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseRestartSentinelContinuation(value: unknown): RestartSentinelContinuation | null {
|
||||
if (!isPlainRecord(value)) {
|
||||
return null;
|
||||
}
|
||||
if (value.kind === "systemEvent" && typeof value.text === "string") {
|
||||
return { kind: "systemEvent", text: value.text };
|
||||
}
|
||||
if (value.kind === "agentTurn" && typeof value.message === "string") {
|
||||
return { kind: "agentTurn", message: value.message };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseRestartSentinelPayload(value: unknown): RestartSentinelPayload | null {
|
||||
if (
|
||||
!isPlainRecord(value) ||
|
||||
!RESTART_SENTINEL_KINDS.has(value.kind as RestartSentinelPayload["kind"]) ||
|
||||
!RESTART_SENTINEL_STATUSES.has(value.status as RestartSentinelPayload["status"]) ||
|
||||
!isSafeInteger(value.ts)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const sessionKey = parseOptionalNullableString(value, "sessionKey");
|
||||
const threadId = parseOptionalNullableString(value, "threadId");
|
||||
const message = parseOptionalNullableString(value, "message");
|
||||
const doctorHint = parseOptionalNullableString(value, "doctorHint");
|
||||
if (
|
||||
sessionKey === false ||
|
||||
sessionKey === null ||
|
||||
threadId === false ||
|
||||
threadId === null ||
|
||||
message === false ||
|
||||
doctorHint === false
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let deliveryContext: RestartSentinelPayload["deliveryContext"];
|
||||
if (value.deliveryContext !== undefined) {
|
||||
if (!isPlainRecord(value.deliveryContext)) {
|
||||
return null;
|
||||
}
|
||||
const channel = parseOptionalNullableString(value.deliveryContext, "channel");
|
||||
const to = parseOptionalNullableString(value.deliveryContext, "to");
|
||||
const accountId = parseOptionalNullableString(value.deliveryContext, "accountId");
|
||||
if (
|
||||
channel === false ||
|
||||
channel === null ||
|
||||
to === false ||
|
||||
to === null ||
|
||||
accountId === false ||
|
||||
accountId === null
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
deliveryContext = {};
|
||||
if (channel !== undefined) {
|
||||
deliveryContext.channel = channel;
|
||||
}
|
||||
if (to !== undefined) {
|
||||
deliveryContext.to = to;
|
||||
}
|
||||
if (accountId !== undefined) {
|
||||
deliveryContext.accountId = accountId;
|
||||
}
|
||||
}
|
||||
|
||||
let continuation: RestartSentinelContinuation | null | undefined;
|
||||
if (value.continuation !== undefined) {
|
||||
continuation =
|
||||
value.continuation === null ? null : parseRestartSentinelContinuation(value.continuation);
|
||||
if (continuation === null && value.continuation !== null) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
let stats: RestartSentinelStats | null | undefined;
|
||||
if (value.stats !== undefined) {
|
||||
stats = value.stats === null ? null : parseRestartSentinelStats(value.stats);
|
||||
if (stats === null && value.stats !== null) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const result: RestartSentinelPayload = {
|
||||
kind: value.kind as RestartSentinelPayload["kind"],
|
||||
status: value.status as RestartSentinelPayload["status"],
|
||||
ts: value.ts,
|
||||
};
|
||||
if (sessionKey !== undefined) {
|
||||
result.sessionKey = sessionKey;
|
||||
}
|
||||
// SQL NULL is canonical absence for optional top-level columns. Normalize
|
||||
// legacy nulls and empty routes so writes and typed-column reads agree.
|
||||
if (deliveryContext !== undefined && Object.keys(deliveryContext).length > 0) {
|
||||
result.deliveryContext = deliveryContext;
|
||||
}
|
||||
if (threadId !== undefined) {
|
||||
result.threadId = threadId;
|
||||
}
|
||||
if (message !== undefined && message !== null) {
|
||||
result.message = message;
|
||||
}
|
||||
if (continuation !== undefined && continuation !== null) {
|
||||
result.continuation = continuation;
|
||||
}
|
||||
if (doctorHint !== undefined && doctorHint !== null) {
|
||||
result.doctorHint = doctorHint;
|
||||
}
|
||||
if (stats !== undefined && stats !== null) {
|
||||
result.stats = stats;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function parseRestartSentinelEnvelope(value: unknown): RestartSentinelEnvelope | null {
|
||||
if (!isPlainRecord(value) || value.version !== 1) {
|
||||
return null;
|
||||
}
|
||||
const payload = parseRestartSentinelPayload(value.payload);
|
||||
return payload ? { version: 1, payload } : null;
|
||||
}
|
||||
|
||||
function parseRequiredJson(value: string | null): unknown {
|
||||
if (value === null) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(value) as unknown;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeRestartSentinelRow(row: {
|
||||
version: number;
|
||||
kind: string;
|
||||
status: string;
|
||||
ts: number;
|
||||
session_key: string | null;
|
||||
thread_id: string | null;
|
||||
delivery_channel: string | null;
|
||||
delivery_to: string | null;
|
||||
delivery_account_id: string | null;
|
||||
message: string | null;
|
||||
continuation_json: string | null;
|
||||
doctor_hint: string | null;
|
||||
stats_json: string | null;
|
||||
updated_at_ms: number;
|
||||
}): RestartSentinel | null {
|
||||
if (row.version !== 1 || !isSafeInteger(row.updated_at_ms)) {
|
||||
return null;
|
||||
}
|
||||
const candidate: Record<string, unknown> = {
|
||||
kind: row.kind,
|
||||
status: row.status,
|
||||
ts: row.ts,
|
||||
};
|
||||
if (row.session_key !== null) {
|
||||
candidate.sessionKey = row.session_key;
|
||||
}
|
||||
if (row.thread_id !== null) {
|
||||
candidate.threadId = row.thread_id;
|
||||
}
|
||||
if (
|
||||
row.delivery_channel !== null ||
|
||||
row.delivery_to !== null ||
|
||||
row.delivery_account_id !== null
|
||||
) {
|
||||
candidate.deliveryContext = {
|
||||
...(row.delivery_channel === null ? {} : { channel: row.delivery_channel }),
|
||||
...(row.delivery_to === null ? {} : { to: row.delivery_to }),
|
||||
...(row.delivery_account_id === null ? {} : { accountId: row.delivery_account_id }),
|
||||
};
|
||||
}
|
||||
if (row.message !== null) {
|
||||
candidate.message = row.message;
|
||||
}
|
||||
if (row.continuation_json !== null) {
|
||||
const continuation = parseRequiredJson(row.continuation_json);
|
||||
if (continuation === undefined) {
|
||||
return null;
|
||||
}
|
||||
candidate.continuation = continuation;
|
||||
}
|
||||
if (row.doctor_hint !== null) {
|
||||
candidate.doctorHint = row.doctor_hint;
|
||||
}
|
||||
if (row.stats_json !== null) {
|
||||
const stats = parseRequiredJson(row.stats_json);
|
||||
if (stats === undefined) {
|
||||
return null;
|
||||
}
|
||||
candidate.stats = stats;
|
||||
}
|
||||
const payload = parseRestartSentinelPayload(candidate);
|
||||
return payload ? { version: 1, payload, revision: row.updated_at_ms } : null;
|
||||
}
|
||||
|
||||
export function readRestartSentinelRowSync(db: DatabaseSync): RestartSentinelRowState {
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select([
|
||||
"version",
|
||||
"kind",
|
||||
"status",
|
||||
"ts",
|
||||
"session_key",
|
||||
"thread_id",
|
||||
"delivery_channel",
|
||||
"delivery_to",
|
||||
"delivery_account_id",
|
||||
"message",
|
||||
"continuation_json",
|
||||
"doctor_hint",
|
||||
"stats_json",
|
||||
"updated_at_ms",
|
||||
])
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
|
||||
);
|
||||
if (!row) {
|
||||
return { kind: "missing" };
|
||||
}
|
||||
const sentinel = decodeRestartSentinelRow(row);
|
||||
return sentinel ? { kind: "valid", sentinel } : { kind: "invalid", revision: row.updated_at_ms };
|
||||
}
|
||||
|
||||
function requireValidPayload(payload: RestartSentinelPayload): RestartSentinelPayload {
|
||||
const parsed = parseRestartSentinelPayload(payload);
|
||||
if (!parsed) {
|
||||
throw new TypeError("Invalid restart sentinel payload");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function nextRevision(currentRevision: number | null): number {
|
||||
if (currentRevision !== null && !Number.isSafeInteger(currentRevision)) {
|
||||
throw new Error("Restart sentinel revision is outside the safe integer range");
|
||||
}
|
||||
// Same-millisecond replacements still need distinct revisions, or a stale
|
||||
// consumer could delete the newer singleton row after delivering the old one.
|
||||
const revision = Math.max(Date.now(), currentRevision === null ? 0 : currentRevision + 1);
|
||||
if (!Number.isSafeInteger(revision)) {
|
||||
throw new Error("Restart sentinel revision exhausted the safe integer range");
|
||||
}
|
||||
return revision;
|
||||
}
|
||||
|
||||
function readRestartSentinelRevisionFloorSync(db: DatabaseSync): number | null {
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select("updated_at_ms")
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_REVISION_FLOOR_KEY),
|
||||
);
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
if (!Number.isSafeInteger(row.updated_at_ms)) {
|
||||
throw new Error("Restart sentinel revision floor is outside the safe integer range");
|
||||
}
|
||||
return row.updated_at_ms;
|
||||
}
|
||||
|
||||
function maxRevision(left: number | null, right: number | null): number | null {
|
||||
if (left === null) {
|
||||
return right;
|
||||
}
|
||||
if (right === null) {
|
||||
return left;
|
||||
}
|
||||
return Math.max(left, right);
|
||||
}
|
||||
|
||||
function buildRestartSentinelRow(
|
||||
payload: RestartSentinelPayload,
|
||||
revision: number,
|
||||
sentinelKey = RESTART_SENTINEL_KEY,
|
||||
) {
|
||||
return {
|
||||
sentinel_key: sentinelKey,
|
||||
version: 1,
|
||||
kind: payload.kind,
|
||||
status: payload.status,
|
||||
ts: payload.ts,
|
||||
session_key: payload.sessionKey ?? null,
|
||||
thread_id: payload.threadId ?? null,
|
||||
delivery_channel: payload.deliveryContext?.channel ?? null,
|
||||
delivery_to: payload.deliveryContext?.to ?? null,
|
||||
delivery_account_id: payload.deliveryContext?.accountId ?? null,
|
||||
message: payload.message ?? null,
|
||||
continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null,
|
||||
doctor_hint: payload.doctorHint ?? null,
|
||||
stats_json: payload.stats ? JSON.stringify(payload.stats) : null,
|
||||
// Debug shadow only. Reads reconstruct exclusively from typed columns above.
|
||||
payload_json: JSON.stringify(payload),
|
||||
updated_at_ms: revision,
|
||||
};
|
||||
}
|
||||
|
||||
function upsertRestartSentinelRowSync(
|
||||
db: DatabaseSync,
|
||||
row: ReturnType<typeof buildRestartSentinelRow>,
|
||||
): void {
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.insertInto("gateway_restart_sentinel")
|
||||
.values(row)
|
||||
.onConflict((conflict) =>
|
||||
conflict.column("sentinel_key").doUpdateSet({
|
||||
version: (eb) => eb.ref("excluded.version"),
|
||||
kind: (eb) => eb.ref("excluded.kind"),
|
||||
status: (eb) => eb.ref("excluded.status"),
|
||||
ts: (eb) => eb.ref("excluded.ts"),
|
||||
session_key: (eb) => eb.ref("excluded.session_key"),
|
||||
thread_id: (eb) => eb.ref("excluded.thread_id"),
|
||||
delivery_channel: (eb) => eb.ref("excluded.delivery_channel"),
|
||||
delivery_to: (eb) => eb.ref("excluded.delivery_to"),
|
||||
delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"),
|
||||
message: (eb) => eb.ref("excluded.message"),
|
||||
continuation_json: (eb) => eb.ref("excluded.continuation_json"),
|
||||
doctor_hint: (eb) => eb.ref("excluded.doctor_hint"),
|
||||
stats_json: (eb) => eb.ref("excluded.stats_json"),
|
||||
payload_json: (eb) => eb.ref("excluded.payload_json"),
|
||||
updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"),
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function advanceRestartSentinelRevisionFloorSync(db: DatabaseSync, revision: number): void {
|
||||
// `current` is deleted after durable delivery. The reserved row survives that
|
||||
// clear so a later same-millisecond write cannot reuse an idempotency revision.
|
||||
const payload: RestartSentinelPayload = { kind: "restart", status: "skipped", ts: revision };
|
||||
upsertRestartSentinelRowSync(
|
||||
db,
|
||||
buildRestartSentinelRow(payload, revision, RESTART_SENTINEL_REVISION_FLOOR_KEY),
|
||||
);
|
||||
}
|
||||
|
||||
export function writeRestartSentinelRowSync(
|
||||
db: DatabaseSync,
|
||||
rawPayload: RestartSentinelPayload,
|
||||
): RestartSentinel {
|
||||
const payload = requireValidPayload(rawPayload);
|
||||
const current = readRestartSentinelRowSync(db);
|
||||
const currentRevision =
|
||||
current.kind === "missing"
|
||||
? null
|
||||
: current.kind === "valid"
|
||||
? current.sentinel.revision
|
||||
: current.revision;
|
||||
const revision = nextRevision(
|
||||
maxRevision(currentRevision, readRestartSentinelRevisionFloorSync(db)),
|
||||
);
|
||||
const row = buildRestartSentinelRow(payload, revision);
|
||||
upsertRestartSentinelRowSync(db, row);
|
||||
advanceRestartSentinelRevisionFloorSync(db, revision);
|
||||
return { version: 1, payload, revision };
|
||||
}
|
||||
|
||||
export function writeRestartSentinelRowIfRevisionSync(
|
||||
db: DatabaseSync,
|
||||
rawPayload: RestartSentinelPayload,
|
||||
expectedRevision: number,
|
||||
): RestartSentinel | null {
|
||||
const current = readRestartSentinelRowSync(db);
|
||||
if (current.kind !== "valid" || current.sentinel.revision !== expectedRevision) {
|
||||
return null;
|
||||
}
|
||||
const payload = requireValidPayload(rawPayload);
|
||||
const revision = nextRevision(
|
||||
maxRevision(expectedRevision, readRestartSentinelRevisionFloorSync(db)),
|
||||
);
|
||||
const row = buildRestartSentinelRow(payload, revision);
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.updateTable("gateway_restart_sentinel")
|
||||
.set(row)
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY)
|
||||
.where("updated_at_ms", "=", expectedRevision),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
return null;
|
||||
}
|
||||
advanceRestartSentinelRevisionFloorSync(db, revision);
|
||||
return { version: 1, payload, revision };
|
||||
}
|
||||
|
||||
export function deleteRestartSentinelRowSync(db: DatabaseSync, expectedRevision?: number): boolean {
|
||||
const current = readRestartSentinelRowSync(db);
|
||||
if (current.kind === "missing") {
|
||||
return false;
|
||||
}
|
||||
const currentRevision = current.kind === "valid" ? current.sentinel.revision : current.revision;
|
||||
if (expectedRevision !== undefined && currentRevision !== expectedRevision) {
|
||||
return false;
|
||||
}
|
||||
if (!Number.isSafeInteger(currentRevision)) {
|
||||
throw new Error("Restart sentinel revision is outside the safe integer range");
|
||||
}
|
||||
advanceRestartSentinelRevisionFloorSync(
|
||||
db,
|
||||
maxRevision(currentRevision, readRestartSentinelRevisionFloorSync(db)) ?? currentRevision,
|
||||
);
|
||||
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
let query = stateDb
|
||||
.deleteFrom("gateway_restart_sentinel")
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY);
|
||||
if (expectedRevision !== undefined) {
|
||||
query = query.where("updated_at_ms", "=", expectedRevision);
|
||||
}
|
||||
if (executeSqliteQuerySync(db, query).numAffectedRows !== 1n) {
|
||||
// The outer write transaction owns both rows; fail closed so its rollback
|
||||
// cannot leave a floor for a current row this call did not consume.
|
||||
throw new Error("Restart sentinel changed during guarded delete");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
+176
-121
@@ -45,6 +45,7 @@ import {
|
||||
import {
|
||||
buildRestartSuccessContinuation,
|
||||
clearRestartSentinel,
|
||||
clearRestartSentinelIfRevision,
|
||||
finalizeUpdateRestartSentinelRunningVersion,
|
||||
formatDoctorNonInteractiveHint,
|
||||
formatRestartSentinelMessage,
|
||||
@@ -87,34 +88,51 @@ function readSentinelRow() {
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select(["sentinel_key", "version", "kind", "status", "payload_json"])
|
||||
.select(["sentinel_key", "version", "kind", "status", "payload_json", "updated_at_ms"])
|
||||
.where("sentinel_key", "=", "current"),
|
||||
);
|
||||
}
|
||||
|
||||
function insertSentinelRow(values: { version?: number; payloadJson: string }) {
|
||||
function readSentinelRevisionFloor() {
|
||||
const { db } = openOpenClawStateDatabase();
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
return executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select("updated_at_ms")
|
||||
.where("sentinel_key", "=", "revision-floor"),
|
||||
)?.updated_at_ms;
|
||||
}
|
||||
|
||||
function deleteSentinelRevisionFloor() {
|
||||
const { db } = openOpenClawStateDatabase();
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb.insertInto("gateway_restart_sentinel").values({
|
||||
sentinel_key: "current",
|
||||
version: values.version ?? 1,
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts: Date.now(),
|
||||
session_key: null,
|
||||
thread_id: null,
|
||||
delivery_channel: null,
|
||||
delivery_to: null,
|
||||
delivery_account_id: null,
|
||||
message: null,
|
||||
continuation_json: null,
|
||||
doctor_hint: null,
|
||||
stats_json: null,
|
||||
payload_json: values.payloadJson,
|
||||
updated_at_ms: Date.now(),
|
||||
}),
|
||||
stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "revision-floor"),
|
||||
);
|
||||
}
|
||||
|
||||
function updateSentinelRow(
|
||||
values: Partial<{
|
||||
version: number;
|
||||
kind: string;
|
||||
status: string;
|
||||
continuation_json: string | null;
|
||||
stats_json: string | null;
|
||||
payload_json: string;
|
||||
updated_at_ms: number;
|
||||
}>,
|
||||
) {
|
||||
const { db } = openOpenClawStateDatabase();
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.updateTable("gateway_restart_sentinel")
|
||||
.set(values)
|
||||
.where("sentinel_key", "=", "current"),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -147,7 +165,28 @@ describe("restart sentinel", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("imports a legacy file sentinel into sqlite once", async () => {
|
||||
it("canonicalizes nullable top-level fields and empty delivery context", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const written = await writeRestartSentinel({
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 1,
|
||||
deliveryContext: {},
|
||||
message: null,
|
||||
continuation: null,
|
||||
doctorHint: null,
|
||||
stats: null,
|
||||
});
|
||||
|
||||
expect(written.payload).toEqual({ kind: "restart", status: "ok", ts: 1 });
|
||||
await expect(readRestartSentinel()).resolves.toEqual(written);
|
||||
expect(readSentinelRow()?.payload_json).toBe(
|
||||
JSON.stringify({ kind: "restart", status: "ok", ts: 1 }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores legacy files without mutating them", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const payload = {
|
||||
kind: "update" as const,
|
||||
@@ -161,100 +200,132 @@ describe("restart sentinel", () => {
|
||||
},
|
||||
};
|
||||
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
|
||||
await fs.writeFile(legacyPath, `${JSON.stringify({ version: 1, payload })}\n`, "utf-8");
|
||||
|
||||
await expect(hasRestartSentinel()).resolves.toBe(true);
|
||||
expect(readSentinelRow()).toMatchObject({
|
||||
sentinel_key: "current",
|
||||
version: 1,
|
||||
kind: "update",
|
||||
status: "skipped",
|
||||
payload_json: JSON.stringify(payload),
|
||||
});
|
||||
await expect(fs.access(legacyPath)).rejects.toThrow();
|
||||
await expect(readRestartSentinel()).resolves.toEqual({ version: 1, payload });
|
||||
});
|
||||
});
|
||||
|
||||
it("does not replay a legacy file superseded by a sqlite sentinel", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
|
||||
await fs.writeFile(
|
||||
legacyPath,
|
||||
`${JSON.stringify({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts: 1,
|
||||
message: "stale legacy sentinel",
|
||||
},
|
||||
})}\n`,
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
await writeRestartSentinel({
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 2,
|
||||
message: "current sqlite sentinel",
|
||||
});
|
||||
await expect(fs.access(legacyPath)).rejects.toThrow();
|
||||
|
||||
await clearRestartSentinel();
|
||||
const legacyContents = `${JSON.stringify({ version: 1, payload })}\n`;
|
||||
await fs.writeFile(legacyPath, legacyContents, "utf-8");
|
||||
|
||||
await expect(hasRestartSentinel()).resolves.toBe(false);
|
||||
await expect(readRestartSentinel()).resolves.toBeNull();
|
||||
await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 });
|
||||
await clearRestartSentinel();
|
||||
await expect(fs.readFile(legacyPath, "utf-8")).resolves.toBe(legacyContents);
|
||||
});
|
||||
});
|
||||
|
||||
it("drops invalid sentinel payloads", async () => {
|
||||
it("reconstructs typed columns when payload_json is corrupt", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
insertSentinelRow({ payloadJson: "not-json" });
|
||||
const payload = {
|
||||
kind: "update" as const,
|
||||
status: "skipped" as const,
|
||||
ts: 42,
|
||||
sessionKey: "agent:main:webchat:dm:user-123",
|
||||
deliveryContext: { channel: "webchat", to: "user-123", accountId: "default" },
|
||||
threadId: "thread-1",
|
||||
message: "typed state",
|
||||
continuation: { kind: "agentTurn" as const, message: "continue" },
|
||||
doctorHint: "run doctor",
|
||||
stats: { mode: "npm", reason: "pending" },
|
||||
};
|
||||
const written = await writeRestartSentinel(payload);
|
||||
updateSentinelRow({ payload_json: "not-json" });
|
||||
|
||||
const read = await readRestartSentinel();
|
||||
expect(read).toBeNull();
|
||||
|
||||
expect(readSentinelRow()).toBeUndefined();
|
||||
await expect(readRestartSentinel()).resolves.toEqual(written);
|
||||
});
|
||||
});
|
||||
|
||||
it("drops structurally invalid sentinel payloads", async () => {
|
||||
it("leaves malformed typed rows in place and reports them as unreadable", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
insertSentinelRow({ version: 2, payloadJson: JSON.stringify(null) });
|
||||
await writeRestartSentinel({ kind: "update", status: "ok", ts: 1 });
|
||||
updateSentinelRow({ kind: "not-a-kind", payload_json: "{}" });
|
||||
|
||||
await expect(readRestartSentinel()).resolves.toBeNull();
|
||||
expect(readSentinelRow()).toBeUndefined();
|
||||
await expect(hasRestartSentinel()).resolves.toBe(false);
|
||||
expect(readSentinelRow()).toMatchObject({ kind: "not-a-kind", payload_json: "{}" });
|
||||
expect(mockWarn).toHaveBeenCalledWith("Ignoring invalid typed restart sentinel row");
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps old config restart sentinels readable without restart-required stats", async () => {
|
||||
it("rejects malformed typed JSON columns even when the shadow payload is valid", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const filePath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
|
||||
const payload = {
|
||||
kind: "config-patch" as const,
|
||||
status: "ok" as const,
|
||||
ts: Date.now(),
|
||||
message: "Config updated successfully",
|
||||
stats: { mode: "config.patch" },
|
||||
};
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, JSON.stringify({ version: 1, payload }, null, 2), "utf-8");
|
||||
const payload = { kind: "update" as const, status: "ok" as const, ts: 1 };
|
||||
await writeRestartSentinel(payload);
|
||||
updateSentinelRow({
|
||||
continuation_json: JSON.stringify({ kind: "agentTurn", message: 42 }),
|
||||
payload_json: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
const read = await readRestartSentinel();
|
||||
await expect(readRestartSentinel()).resolves.toBeNull();
|
||||
await expect(hasRestartSentinel()).resolves.toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
expect(read?.payload).toEqual(payload);
|
||||
if (!read) {
|
||||
throw new Error("Expected old restart sentinel to be readable");
|
||||
it("keeps revisions strictly monotonic within the same millisecond", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const now = vi.spyOn(Date, "now").mockReturnValue(1000);
|
||||
try {
|
||||
const first = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 });
|
||||
const second = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 });
|
||||
expect(second.revision).toBe(first.revision + 1);
|
||||
expect(readSentinelRow()?.updated_at_ms).toBe(second.revision);
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
expect(summarizeRestartSentinel(read.payload)).toBe(
|
||||
"Gateway restart config-patch ok (config.patch)",
|
||||
);
|
||||
expect(formatRestartSentinelMessage(read.payload)).toBe(
|
||||
["Gateway restart config-patch ok (config.patch)", "Config updated successfully"].join(
|
||||
"\n",
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("upgrades pre-floor rows before unconditional and guarded clears", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const now = vi.spyOn(Date, "now").mockReturnValue(1000);
|
||||
try {
|
||||
const first = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 });
|
||||
deleteSentinelRevisionFloor();
|
||||
expect(readSentinelRevisionFloor()).toBeUndefined();
|
||||
await expect(clearRestartSentinel()).resolves.toBe(true);
|
||||
|
||||
await expect(readRestartSentinel()).resolves.toBeNull();
|
||||
await expect(hasRestartSentinel()).resolves.toBe(false);
|
||||
expect(readSentinelRevisionFloor()).toBe(first.revision);
|
||||
|
||||
now.mockReturnValue(500);
|
||||
const second = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 2 });
|
||||
expect(second.revision).toBe(first.revision + 1);
|
||||
|
||||
deleteSentinelRevisionFloor();
|
||||
await expect(clearRestartSentinelIfRevision(second.revision + 1)).resolves.toBe(false);
|
||||
expect(readSentinelRevisionFloor()).toBeUndefined();
|
||||
await expect(readRestartSentinel()).resolves.toEqual(second);
|
||||
|
||||
await expect(clearRestartSentinelIfRevision(second.revision)).resolves.toBe(true);
|
||||
expect(readSentinelRevisionFloor()).toBe(second.revision);
|
||||
const third = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 3 });
|
||||
expect(third.revision).toBe(second.revision + 1);
|
||||
|
||||
await expect(clearRestartSentinelIfRevision(third.revision)).resolves.toBe(true);
|
||||
deleteSentinelRevisionFloor();
|
||||
await expect(clearRestartSentinel()).resolves.toBe(false);
|
||||
expect(readSentinelRevisionFloor()).toBeUndefined();
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let stale deletes remove a newer sentinel", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const first = await writeRestartSentinel({
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 1,
|
||||
message: "old",
|
||||
});
|
||||
const newer = await writeRestartSentinel({
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 2,
|
||||
message: "new",
|
||||
});
|
||||
|
||||
await expect(clearRestartSentinelIfRevision(first.revision)).resolves.toBe(false);
|
||||
await expect(readRestartSentinel()).resolves.toEqual(newer);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -422,7 +493,7 @@ describe("restart sentinel", () => {
|
||||
|
||||
await finalizeUpdateRestartSentinelRunningVersion("actual-version");
|
||||
|
||||
await expect(readRestartSentinel()).resolves.toEqual({
|
||||
await expect(readRestartSentinel()).resolves.toMatchObject({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
@@ -453,7 +524,7 @@ describe("restart sentinel", () => {
|
||||
await expect(
|
||||
finalizeUpdateRestartSentinelRunningVersion("actual-version"),
|
||||
).resolves.toBeNull();
|
||||
await expect(readRestartSentinel()).resolves.toEqual({
|
||||
await expect(readRestartSentinel()).resolves.toMatchObject({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
@@ -481,7 +552,7 @@ describe("restart sentinel", () => {
|
||||
|
||||
await markUpdateRestartSentinelFailure("restart-unhealthy");
|
||||
|
||||
await expect(readRestartSentinel()).resolves.toEqual({
|
||||
await expect(readRestartSentinel()).resolves.toMatchObject({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
@@ -497,18 +568,16 @@ describe("restart sentinel", () => {
|
||||
});
|
||||
|
||||
describe("restart sentinel error visibility", () => {
|
||||
it("logs a warning when clearRestartSentinel DB write fails", async () => {
|
||||
mockThrowWrite.mockImplementationOnce(() => {
|
||||
throw new Error("SQLITE_IOERR: disk I/O error");
|
||||
});
|
||||
|
||||
it("throws when clearRestartSentinel cannot durably delete the row", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
await expect(clearRestartSentinel()).resolves.toBeUndefined();
|
||||
const written = await writeRestartSentinel({ kind: "restart", status: "ok", ts: 1 });
|
||||
mockThrowWrite.mockImplementationOnce(() => {
|
||||
throw new Error("SQLITE_IOERR: disk I/O error");
|
||||
});
|
||||
|
||||
expect(mockWarn).toHaveBeenCalledTimes(1);
|
||||
expect(mockWarn).toHaveBeenCalledWith(
|
||||
"Failed to clear restart sentinel: SQLITE_IOERR: disk I/O error",
|
||||
);
|
||||
await expect(clearRestartSentinel()).rejects.toThrow("SQLITE_IOERR: disk I/O error");
|
||||
expect(mockWarn).not.toHaveBeenCalled();
|
||||
await expect(readRestartSentinel()).resolves.toEqual(written);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -541,20 +610,6 @@ describe("restart sentinel error visibility", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("logs a warning when the legacy sentinel path cannot be removed", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
|
||||
await fs.mkdir(legacyPath);
|
||||
|
||||
await expect(clearRestartSentinel()).resolves.toBeUndefined();
|
||||
|
||||
expect(mockWarn).toHaveBeenCalledTimes(1);
|
||||
expect(mockWarn).toHaveBeenCalledWith(
|
||||
expect.stringMatching(/^Failed to remove legacy restart sentinel: .+/),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("restart success continuation", () => {
|
||||
|
||||
+54
-217
@@ -1,12 +1,8 @@
|
||||
// Persists restart sentinel state that coordinates deferred restarts.
|
||||
import { readFile, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { formatCliCommand } from "../cli/command-format.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
runOpenClawStateWriteTransaction,
|
||||
@@ -14,77 +10,22 @@ import {
|
||||
import { resolveRuntimeServiceVersion } from "../version.js";
|
||||
import { formatErrorMessage } from "./errors.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "./kysely-sync.js";
|
||||
deleteRestartSentinelRowSync,
|
||||
readRestartSentinelRowSync,
|
||||
writeRestartSentinelRowIfRevisionSync,
|
||||
writeRestartSentinelRowSync,
|
||||
type RestartSentinel,
|
||||
type RestartSentinelContinuation,
|
||||
type RestartSentinelPayload,
|
||||
} from "./restart-sentinel-store.js";
|
||||
|
||||
export type {
|
||||
RestartSentinelContinuation,
|
||||
RestartSentinelPayload,
|
||||
} from "./restart-sentinel-store.js";
|
||||
|
||||
const sentinelLog = createSubsystemLogger("restart-sentinel");
|
||||
|
||||
type RestartSentinelLog = {
|
||||
stdoutTail?: string | null;
|
||||
stderrTail?: string | null;
|
||||
exitCode?: number | null;
|
||||
};
|
||||
|
||||
type RestartSentinelStep = {
|
||||
name: string;
|
||||
command: string;
|
||||
cwd?: string | null;
|
||||
durationMs?: number | null;
|
||||
log?: RestartSentinelLog | null;
|
||||
};
|
||||
|
||||
type RestartSentinelStats = {
|
||||
mode?: string;
|
||||
root?: string;
|
||||
requiresRestart?: boolean;
|
||||
handoffId?: string;
|
||||
before?: Record<string, unknown> | null;
|
||||
after?: Record<string, unknown> | null;
|
||||
steps?: RestartSentinelStep[];
|
||||
reason?: string | null;
|
||||
durationMs?: number | null;
|
||||
};
|
||||
|
||||
export type RestartSentinelContinuation =
|
||||
| {
|
||||
kind: "systemEvent";
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
kind: "agentTurn";
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type RestartSentinelPayload = {
|
||||
kind: "config-apply" | "config-auto-recovery" | "config-patch" | "update" | "restart";
|
||||
status: "ok" | "error" | "skipped";
|
||||
ts: number;
|
||||
sessionKey?: string;
|
||||
/** Delivery context captured at restart time to ensure channel routing survives restart. */
|
||||
deliveryContext?: {
|
||||
channel?: string;
|
||||
to?: string;
|
||||
accountId?: string;
|
||||
};
|
||||
/** Thread ID for reply threading (e.g., Slack thread_ts). */
|
||||
threadId?: string;
|
||||
message?: string | null;
|
||||
continuation?: RestartSentinelContinuation | null;
|
||||
doctorHint?: string | null;
|
||||
stats?: RestartSentinelStats | null;
|
||||
};
|
||||
|
||||
type RestartSentinel = {
|
||||
version: 1;
|
||||
payload: RestartSentinelPayload;
|
||||
};
|
||||
|
||||
const RESTART_SENTINEL_KEY = "current";
|
||||
const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json";
|
||||
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
|
||||
|
||||
export function formatDoctorNonInteractiveHint(
|
||||
env: Record<string, string | undefined> = process.env as Record<string, string | undefined>,
|
||||
): string {
|
||||
@@ -97,57 +38,12 @@ export function formatDoctorNonInteractiveHint(
|
||||
export async function writeRestartSentinel(
|
||||
payload: RestartSentinelPayload,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<void> {
|
||||
const updatedAtMs = Date.now();
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.insertInto("gateway_restart_sentinel")
|
||||
.values({
|
||||
sentinel_key: RESTART_SENTINEL_KEY,
|
||||
version: 1,
|
||||
kind: payload.kind,
|
||||
status: payload.status,
|
||||
ts: payload.ts,
|
||||
session_key: payload.sessionKey ?? null,
|
||||
thread_id: payload.threadId ?? null,
|
||||
delivery_channel: payload.deliveryContext?.channel ?? null,
|
||||
delivery_to: payload.deliveryContext?.to ?? null,
|
||||
delivery_account_id: payload.deliveryContext?.accountId ?? null,
|
||||
message: payload.message ?? null,
|
||||
continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null,
|
||||
doctor_hint: payload.doctorHint ?? null,
|
||||
stats_json: payload.stats ? JSON.stringify(payload.stats) : null,
|
||||
payload_json: JSON.stringify(payload),
|
||||
updated_at_ms: updatedAtMs,
|
||||
})
|
||||
.onConflict((conflict) =>
|
||||
conflict.column("sentinel_key").doUpdateSet({
|
||||
version: (eb) => eb.ref("excluded.version"),
|
||||
kind: (eb) => eb.ref("excluded.kind"),
|
||||
status: (eb) => eb.ref("excluded.status"),
|
||||
ts: (eb) => eb.ref("excluded.ts"),
|
||||
session_key: (eb) => eb.ref("excluded.session_key"),
|
||||
thread_id: (eb) => eb.ref("excluded.thread_id"),
|
||||
delivery_channel: (eb) => eb.ref("excluded.delivery_channel"),
|
||||
delivery_to: (eb) => eb.ref("excluded.delivery_to"),
|
||||
delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"),
|
||||
message: (eb) => eb.ref("excluded.message"),
|
||||
continuation_json: (eb) => eb.ref("excluded.continuation_json"),
|
||||
doctor_hint: (eb) => eb.ref("excluded.doctor_hint"),
|
||||
stats_json: (eb) => eb.ref("excluded.stats_json"),
|
||||
payload_json: (eb) => eb.ref("excluded.payload_json"),
|
||||
updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"),
|
||||
}),
|
||||
),
|
||||
);
|
||||
},
|
||||
): Promise<RestartSentinel> {
|
||||
return runOpenClawStateWriteTransaction(
|
||||
({ db }) => writeRestartSentinelRowSync(db, payload),
|
||||
{ env },
|
||||
{ operationLabel: "restart-sentinel.write" },
|
||||
);
|
||||
await removeLegacyRestartSentinel(env);
|
||||
}
|
||||
|
||||
function cloneRestartSentinelPayload(payload: RestartSentinelPayload): RestartSentinelPayload {
|
||||
@@ -158,19 +54,20 @@ async function rewriteRestartSentinel(
|
||||
rewrite: (payload: RestartSentinelPayload) => RestartSentinelPayload | null,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<RestartSentinel | null> {
|
||||
const current = await readRestartSentinel(env);
|
||||
if (!current) {
|
||||
return null;
|
||||
}
|
||||
const nextPayload = rewrite(cloneRestartSentinelPayload(current.payload));
|
||||
if (!nextPayload) {
|
||||
return null;
|
||||
}
|
||||
await writeRestartSentinel(nextPayload, env);
|
||||
return {
|
||||
version: 1,
|
||||
payload: nextPayload,
|
||||
};
|
||||
return runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
const current = readRestartSentinelRowSync(db);
|
||||
if (current.kind !== "valid") {
|
||||
return null;
|
||||
}
|
||||
const nextPayload = rewrite(cloneRestartSentinelPayload(current.sentinel.payload));
|
||||
return nextPayload
|
||||
? writeRestartSentinelRowIfRevisionSync(db, nextPayload, current.sentinel.revision)
|
||||
: null;
|
||||
},
|
||||
{ env },
|
||||
{ operationLabel: "restart-sentinel.rewrite-current" },
|
||||
);
|
||||
}
|
||||
|
||||
export async function finalizeUpdateRestartSentinelRunningVersion(
|
||||
@@ -215,60 +112,23 @@ export async function markUpdateRestartSentinelFailure(
|
||||
}, env);
|
||||
}
|
||||
|
||||
export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<void> {
|
||||
try {
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.deleteFrom("gateway_restart_sentinel")
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
|
||||
);
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
} catch (err) {
|
||||
// Clearing the sentinel is best-effort during shutdown/cleanup, but a
|
||||
// failure here may leave the gateway believing a restart is still pending.
|
||||
sentinelLog.warn(`Failed to clear restart sentinel: ${formatErrorMessage(err)}`);
|
||||
}
|
||||
await removeLegacyRestartSentinel(env);
|
||||
export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
|
||||
return runOpenClawStateWriteTransaction(
|
||||
({ db }) => deleteRestartSentinelRowSync(db),
|
||||
{ env },
|
||||
{ operationLabel: "restart-sentinel.clear" },
|
||||
);
|
||||
}
|
||||
|
||||
function resolveLegacyRestartSentinelPath(env: NodeJS.ProcessEnv): string {
|
||||
return path.join(resolveStateDir(env), LEGACY_RESTART_SENTINEL_FILENAME);
|
||||
}
|
||||
|
||||
async function removeLegacyRestartSentinel(env: NodeJS.ProcessEnv): Promise<void> {
|
||||
try {
|
||||
await rm(resolveLegacyRestartSentinelPath(env), { force: true });
|
||||
} catch (err) {
|
||||
// Legacy cleanup must not block the canonical SQLite operation, but a
|
||||
// failed removal can replay stale restart state after the database clears.
|
||||
sentinelLog.warn(`Failed to remove legacy restart sentinel: ${formatErrorMessage(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function importLegacyRestartSentinel(
|
||||
export async function clearRestartSentinelIfRevision(
|
||||
expectedRevision: number,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<RestartSentinel | null> {
|
||||
const legacyPath = resolveLegacyRestartSentinelPath(env);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(await readFile(legacyPath, "utf-8")) as unknown;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!isPlainRecord(parsed) || parsed.version !== 1 || !isPlainRecord(parsed.payload)) {
|
||||
await removeLegacyRestartSentinel(env);
|
||||
return null;
|
||||
}
|
||||
const payload = parsed.payload as RestartSentinelPayload;
|
||||
await writeRestartSentinel(payload, env);
|
||||
await removeLegacyRestartSentinel(env);
|
||||
return { version: 1, payload };
|
||||
): Promise<boolean> {
|
||||
return runOpenClawStateWriteTransaction(
|
||||
({ db }) => deleteRestartSentinelRowSync(db, expectedRevision),
|
||||
{ env },
|
||||
{ operationLabel: "restart-sentinel.clear-if-revision" },
|
||||
);
|
||||
}
|
||||
|
||||
export function buildRestartSuccessContinuation(params: {
|
||||
@@ -287,29 +147,12 @@ export async function readRestartSentinel(
|
||||
): Promise<RestartSentinel | null> {
|
||||
try {
|
||||
const database = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(database.db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select(["version", "payload_json"])
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
|
||||
);
|
||||
if (!row) {
|
||||
return await importLegacyRestartSentinel(env);
|
||||
}
|
||||
let payload: RestartSentinelPayload | undefined;
|
||||
try {
|
||||
payload = JSON.parse(row.payload_json) as RestartSentinelPayload | undefined;
|
||||
} catch {
|
||||
await clearRestartSentinel(env);
|
||||
const current = readRestartSentinelRowSync(database.db);
|
||||
if (current.kind === "invalid") {
|
||||
sentinelLog.warn("Ignoring invalid typed restart sentinel row");
|
||||
return null;
|
||||
}
|
||||
if (row.version !== 1 || !payload) {
|
||||
await clearRestartSentinel(env);
|
||||
return null;
|
||||
}
|
||||
return { version: 1, payload };
|
||||
return current.kind === "valid" ? current.sentinel : null;
|
||||
} catch (err) {
|
||||
sentinelLog.warn(`Failed to read restart sentinel: ${formatErrorMessage(err)}`);
|
||||
return null;
|
||||
@@ -319,18 +162,12 @@ export async function readRestartSentinel(
|
||||
export async function hasRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
|
||||
try {
|
||||
const database = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(database.db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select("sentinel_key")
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
|
||||
);
|
||||
if (row) {
|
||||
return true;
|
||||
const current = readRestartSentinelRowSync(database.db);
|
||||
if (current.kind === "invalid") {
|
||||
sentinelLog.warn("Ignoring invalid typed restart sentinel row");
|
||||
return false;
|
||||
}
|
||||
return Boolean(await importLegacyRestartSentinel(env));
|
||||
return current.kind === "valid";
|
||||
} catch (err) {
|
||||
sentinelLog.warn(`Failed to check restart sentinel: ${formatErrorMessage(err)}`);
|
||||
return false;
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
moveDeliveryQueueEntryToFailed,
|
||||
updateDeliveryQueueEntry,
|
||||
upsertDeliveryQueueEntry,
|
||||
type DeliveryQueueCompletionRetention,
|
||||
type DeliveryQueueRowMetadata,
|
||||
} from "./delivery-queue-sqlite.js";
|
||||
import { generateSecureUuid } from "./secure-random.js";
|
||||
@@ -28,6 +29,8 @@ type SessionDeliveryContext = {
|
||||
|
||||
type SessionDeliveryRetryPolicy = {
|
||||
maxRetries?: number;
|
||||
/** Retain terminal ownership when the durable producer can replay forever. */
|
||||
completionRetention?: DeliveryQueueCompletionRetention;
|
||||
};
|
||||
|
||||
export type SessionDeliveryRoute = {
|
||||
@@ -139,7 +142,9 @@ export async function enqueueSessionDelivery(
|
||||
entry,
|
||||
metadata: queuedSessionDeliveryMetadata(entry),
|
||||
stateDir,
|
||||
reviveFailedOrCorruptPending: Boolean(params.idempotencyKey),
|
||||
...(params.completionRetention === "permanent"
|
||||
? { insertOnly: true }
|
||||
: { reviveFailedOrCorruptPending: Boolean(params.idempotencyKey) }),
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
@@ -129,6 +129,24 @@ describe("session-delivery queue storage", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("never revives a failed permanent producer intent", async () => {
|
||||
await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => {
|
||||
const payload = {
|
||||
kind: "systemEvent" as const,
|
||||
sessionKey: "agent:main:main",
|
||||
text: "restart complete",
|
||||
idempotencyKey: "restart:permanent-failed",
|
||||
completionRetention: "permanent" as const,
|
||||
};
|
||||
const id = await enqueueSessionDelivery(payload, tempDir);
|
||||
await moveSessionDeliveryToFailed(id, tempDir);
|
||||
|
||||
expect(await enqueueSessionDelivery(payload, tempDir)).toBe(id);
|
||||
expect(readSessionQueueStatus(tempDir, id)).toBe("failed");
|
||||
expect(await loadPendingSessionDeliveries(tempDir)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a completed conflict after acknowledgement", async () => {
|
||||
await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => {
|
||||
const payload = {
|
||||
@@ -355,4 +373,22 @@ describe("session-delivery queue storage", () => {
|
||||
expect(readSessionQueueStatus(tempDir, id)).toBe("completed");
|
||||
});
|
||||
});
|
||||
|
||||
it("retains a permanent completion receipt", async () => {
|
||||
await withTempDir({ prefix: "openclaw-session-delivery-" }, async (tempDir) => {
|
||||
const payload = {
|
||||
kind: "systemEvent" as const,
|
||||
sessionKey: "agent:main:main",
|
||||
text: "restart complete",
|
||||
idempotencyKey: "restart:permanent-completed",
|
||||
completionRetention: "permanent" as const,
|
||||
};
|
||||
const id = await enqueueSessionDelivery(payload, tempDir);
|
||||
await settleSessionDelivery(id, tempDir);
|
||||
|
||||
expect(await enqueueSessionDelivery(payload, tempDir)).toBe(id);
|
||||
expect(readSessionQueueStatus(tempDir, id)).toBe("completed");
|
||||
expect(await loadPendingSessionDeliveries(tempDir)).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -79,6 +79,10 @@ import {
|
||||
detectLegacyRescuePending,
|
||||
discardLegacyRescuePending,
|
||||
} from "./state-migrations.rescue-pending.js";
|
||||
import {
|
||||
detectLegacyRestartSentinel,
|
||||
migrateLegacyRestartSentinel,
|
||||
} from "./state-migrations.restart-sentinel.js";
|
||||
import {
|
||||
migrateLegacyConfigHealth,
|
||||
migrateLegacyCurrentConversationBindings,
|
||||
@@ -416,6 +420,7 @@ export async function detectLegacyStateMigrations(params: {
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations,
|
||||
});
|
||||
const restartSentinel = detectLegacyRestartSentinel({ stateDir });
|
||||
const workspace = detectLegacyWorkspaceState({
|
||||
cfg: params.cfg,
|
||||
stateDir,
|
||||
@@ -598,6 +603,9 @@ export async function detectLegacyStateMigrations(params: {
|
||||
if (mcpOauth.hasLegacy) {
|
||||
preview.push("- MCP OAuth credentials: legacy JSON → shared SQLite state");
|
||||
}
|
||||
if (restartSentinel.hasLegacy) {
|
||||
preview.push("- Restart sentinel: legacy JSON → shared SQLite state");
|
||||
}
|
||||
if (workspace.hasLegacy) {
|
||||
preview.push("- Workspace setup and attestations: legacy files → shared SQLite state");
|
||||
}
|
||||
@@ -701,6 +709,7 @@ export async function detectLegacyStateMigrations(params: {
|
||||
managedOutgoingImages,
|
||||
apns,
|
||||
mcpOauth,
|
||||
restartSentinel,
|
||||
workspace,
|
||||
webPush,
|
||||
nodeHost,
|
||||
@@ -909,6 +918,11 @@ export async function runLegacyStateMigrations(params: {
|
||||
env,
|
||||
stateDir: detected.stateDir,
|
||||
});
|
||||
const restartSentinel = await migrateLegacyRestartSentinel({
|
||||
detected: detected.restartSentinel,
|
||||
env,
|
||||
stateDir: detected.stateDir,
|
||||
});
|
||||
const workspace = await migrateLegacyWorkspaceState({
|
||||
detected: detected.workspace,
|
||||
env,
|
||||
@@ -968,6 +982,7 @@ export async function runLegacyStateMigrations(params: {
|
||||
managedOutgoingImages,
|
||||
apns,
|
||||
mcpOauth,
|
||||
restartSentinel,
|
||||
workspace,
|
||||
webPush,
|
||||
nodeHost,
|
||||
@@ -993,6 +1008,7 @@ export async function runLegacyStateMigrations(params: {
|
||||
...managedOutgoingImages.changes,
|
||||
...apns.changes,
|
||||
...mcpOauth.changes,
|
||||
...restartSentinel.changes,
|
||||
...workspace.changes,
|
||||
...webPush.changes,
|
||||
...nodeHost.changes,
|
||||
@@ -1025,6 +1041,7 @@ export async function runLegacyStateMigrations(params: {
|
||||
...managedOutgoingImages.warnings,
|
||||
...apns.warnings,
|
||||
...mcpOauth.warnings,
|
||||
...restartSentinel.warnings,
|
||||
...workspace.warnings,
|
||||
...webPush.warnings,
|
||||
...nodeHost.warnings,
|
||||
@@ -1187,6 +1204,11 @@ export async function autoMigrateLegacyState(params: {
|
||||
detected: detected.currentConversationBindings,
|
||||
stateDir: detected.stateDir,
|
||||
});
|
||||
const restartSentinel = await migrateLegacyRestartSentinel({
|
||||
detected: detected.restartSentinel,
|
||||
env,
|
||||
stateDir: detected.stateDir,
|
||||
});
|
||||
const channelPairing = migrateLegacyChannelPairingState({
|
||||
detected: detected.channelPairing,
|
||||
env: { ...env, OPENCLAW_STATE_DIR: detected.stateDir },
|
||||
@@ -1214,6 +1236,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
...configHealth.changes,
|
||||
...pluginBindingApprovals.changes,
|
||||
...currentConversationBindings.changes,
|
||||
...restartSentinel.changes,
|
||||
...channelPairing.changes,
|
||||
...preSessionChannelPlans.changes,
|
||||
...pluginPlans.changes,
|
||||
@@ -1234,11 +1257,19 @@ export async function autoMigrateLegacyState(params: {
|
||||
...configHealth.warnings,
|
||||
...pluginBindingApprovals.warnings,
|
||||
...currentConversationBindings.warnings,
|
||||
...restartSentinel.warnings,
|
||||
...channelPairing.warnings,
|
||||
...preSessionChannelPlans.warnings,
|
||||
...pluginPlans.warnings,
|
||||
];
|
||||
const noticeSources = [stateDirResult, detected, pluginInstallIndex, updateCheck, pluginPlans];
|
||||
const noticeSources = [
|
||||
stateDirResult,
|
||||
detected,
|
||||
pluginInstallIndex,
|
||||
updateCheck,
|
||||
restartSentinel,
|
||||
pluginPlans,
|
||||
];
|
||||
const notices = mergeNotices(noticeSources);
|
||||
logMigrationResults(changes, warnings, notices);
|
||||
return {
|
||||
@@ -1257,6 +1288,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
configHealth.changes.length > 0 ||
|
||||
pluginBindingApprovals.changes.length > 0 ||
|
||||
currentConversationBindings.changes.length > 0 ||
|
||||
restartSentinel.changes.length > 0 ||
|
||||
channelPairing.changes.length > 0 ||
|
||||
preSessionChannelPlans.changes.length > 0 ||
|
||||
pluginPlans.changes.length > 0,
|
||||
@@ -1282,6 +1314,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
!detected.configHealth.hasLegacy &&
|
||||
!detected.pluginBindingApprovals.hasLegacy &&
|
||||
!detected.currentConversationBindings.hasLegacy &&
|
||||
!detected.restartSentinel?.hasLegacy &&
|
||||
!detected.workspace.hasLegacy &&
|
||||
!detected.channelPairing.hasLegacy
|
||||
) {
|
||||
@@ -1350,6 +1383,11 @@ export async function autoMigrateLegacyState(params: {
|
||||
detected: detected.currentConversationBindings,
|
||||
stateDir: detected.stateDir,
|
||||
});
|
||||
const restartSentinel = await migrateLegacyRestartSentinel({
|
||||
detected: detected.restartSentinel,
|
||||
env,
|
||||
stateDir: detected.stateDir,
|
||||
});
|
||||
const channelPairing = migrateLegacyChannelPairingState({
|
||||
detected: detected.channelPairing,
|
||||
env: { ...env, OPENCLAW_STATE_DIR: detected.stateDir },
|
||||
@@ -1390,6 +1428,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
...configHealth.changes,
|
||||
...pluginBindingApprovals.changes,
|
||||
...currentConversationBindings.changes,
|
||||
...restartSentinel.changes,
|
||||
...channelPairing.changes,
|
||||
...preSessionChannelPlans.changes,
|
||||
...pluginPlans.changes,
|
||||
@@ -1414,6 +1453,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
...configHealth.warnings,
|
||||
...pluginBindingApprovals.warnings,
|
||||
...currentConversationBindings.warnings,
|
||||
...restartSentinel.warnings,
|
||||
...channelPairing.warnings,
|
||||
...preSessionChannelPlans.warnings,
|
||||
...pluginPlans.warnings,
|
||||
@@ -1422,7 +1462,14 @@ export async function autoMigrateLegacyState(params: {
|
||||
...agentDir.warnings,
|
||||
...channelPlans.warnings,
|
||||
];
|
||||
const noticeSources = [stateDirResult, detected, pluginInstallIndex, updateCheck, pluginPlans];
|
||||
const noticeSources = [
|
||||
stateDirResult,
|
||||
detected,
|
||||
pluginInstallIndex,
|
||||
updateCheck,
|
||||
restartSentinel,
|
||||
pluginPlans,
|
||||
];
|
||||
const notices = mergeNotices(noticeSources);
|
||||
|
||||
logMigrationResults(changes, warnings, notices);
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
// Covers safe startup/Doctor import of the retired restart-sentinel JSON file.
|
||||
import fs from "node:fs";
|
||||
import fsp from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { acquireGatewayLock } from "./gateway-lock.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "./kysely-sync.js";
|
||||
import {
|
||||
clearRestartSentinel,
|
||||
readRestartSentinel,
|
||||
writeRestartSentinel,
|
||||
type RestartSentinelPayload,
|
||||
} from "./restart-sentinel.js";
|
||||
import {
|
||||
detectLegacyRestartSentinel,
|
||||
migrateLegacyRestartSentinel,
|
||||
} from "./state-migrations.restart-sentinel.js";
|
||||
|
||||
type MigrationDatabase = Pick<
|
||||
OpenClawStateKyselyDatabase,
|
||||
"gateway_restart_sentinel" | "migration_sources"
|
||||
>;
|
||||
|
||||
describe("legacy restart sentinel migration", () => {
|
||||
const tempDirs = useAutoCleanupTempDirTracker((cleanup) => {
|
||||
afterEach(() => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
cleanup();
|
||||
});
|
||||
});
|
||||
|
||||
function useStateDir(): { env: NodeJS.ProcessEnv; stateDir: string } {
|
||||
const stateDir = tempDirs.make("openclaw-restart-sentinel-migration-");
|
||||
return { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, stateDir };
|
||||
}
|
||||
|
||||
function payload(ts = 123): RestartSentinelPayload {
|
||||
return {
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts,
|
||||
sessionKey: "agent:main:main",
|
||||
deliveryContext: { channel: "test", to: "target", accountId: "default" },
|
||||
threadId: "thread-1",
|
||||
message: "Update completed",
|
||||
continuation: { kind: "agentTurn", message: "Continue after restart" },
|
||||
doctorHint: "Run Doctor",
|
||||
stats: {
|
||||
mode: "managed",
|
||||
handoffId: "handoff-1",
|
||||
requiresRestart: true,
|
||||
before: { version: "old" },
|
||||
after: { version: "new" },
|
||||
steps: [
|
||||
{
|
||||
name: "install",
|
||||
command: "package-manager update",
|
||||
durationMs: 10,
|
||||
log: { stdoutTail: "done", stderrTail: null, exitCode: 0 },
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function writeLegacy(stateDir: string, value: unknown): Promise<string> {
|
||||
const sourcePath = path.join(stateDir, "restart-sentinel.json");
|
||||
await fsp.writeFile(sourcePath, `${JSON.stringify(value)}\n`, "utf8");
|
||||
return sourcePath;
|
||||
}
|
||||
|
||||
async function migrate(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
stateDir: string;
|
||||
beforeVerify?: () => void;
|
||||
removeSource?: (sourcePath: string) => Promise<void> | void;
|
||||
}) {
|
||||
return await migrateLegacyRestartSentinel({
|
||||
detected: detectLegacyRestartSentinel({ stateDir: params.stateDir }),
|
||||
...params,
|
||||
});
|
||||
}
|
||||
|
||||
function database(env: NodeJS.ProcessEnv) {
|
||||
return openOpenClawStateDatabase({ env }).db;
|
||||
}
|
||||
|
||||
function receipt(env: NodeJS.ProcessEnv) {
|
||||
const db = database(env);
|
||||
return executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
getNodeSqliteKysely<MigrationDatabase>(db)
|
||||
.selectFrom("migration_sources")
|
||||
.selectAll()
|
||||
.where("migration_kind", "=", "legacy-restart-sentinel-json"),
|
||||
);
|
||||
}
|
||||
|
||||
it("detects both the retired source and an interrupted fixed claim", async () => {
|
||||
const { stateDir } = useStateDir();
|
||||
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() });
|
||||
expect(detectLegacyRestartSentinel({ stateDir }).hasLegacy).toBe(true);
|
||||
|
||||
await fsp.rename(sourcePath, `${sourcePath}.doctor-importing`);
|
||||
expect(detectLegacyRestartSentinel({ stateDir }).hasLegacy).toBe(true);
|
||||
});
|
||||
|
||||
it("imports and verifies the complete legacy payload before removing the source", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const expected = payload();
|
||||
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: expected });
|
||||
|
||||
const result = await migrate({ env, stateDir });
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.changes).toEqual([
|
||||
"Imported the legacy restart sentinel into shared SQLite state.",
|
||||
]);
|
||||
await expect(readRestartSentinel(env)).resolves.toMatchObject({
|
||||
version: 1,
|
||||
payload: expected,
|
||||
});
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
expect(receipt(env)).toMatchObject({
|
||||
removed_source: 1,
|
||||
source_record_count: 1,
|
||||
status: "completed",
|
||||
target_table: "gateway_restart_sentinel",
|
||||
});
|
||||
});
|
||||
|
||||
it("canonicalizes legacy null fields and an empty delivery context before verification", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const sourcePath = await writeLegacy(stateDir, {
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 123,
|
||||
deliveryContext: {},
|
||||
message: null,
|
||||
continuation: null,
|
||||
doctorHint: null,
|
||||
stats: null,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await migrate({ env, stateDir });
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.changes).toEqual([
|
||||
"Imported the legacy restart sentinel into shared SQLite state.",
|
||||
]);
|
||||
const migrated = await readRestartSentinel(env);
|
||||
expect(migrated?.payload).toEqual({ kind: "restart", status: "ok", ts: 123 });
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
expect(receipt(env)).toMatchObject({
|
||||
removed_source: 1,
|
||||
source_record_count: 1,
|
||||
status: "completed",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a valid canonical row when legacy JSON conflicts", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const canonical = payload(999);
|
||||
await writeRestartSentinel(canonical, env);
|
||||
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(1) });
|
||||
|
||||
const result = await migrate({ env, stateDir });
|
||||
|
||||
expect(result.changes).toEqual([
|
||||
"Preserved the canonical SQLite restart sentinel and discarded conflicting legacy JSON.",
|
||||
]);
|
||||
await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: canonical });
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("repairs an invalid canonical row from a validated legacy envelope", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const db = database(env);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
getNodeSqliteKysely<MigrationDatabase>(db).insertInto("gateway_restart_sentinel").values({
|
||||
sentinel_key: "current",
|
||||
version: 99,
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts: 1,
|
||||
session_key: null,
|
||||
thread_id: null,
|
||||
delivery_channel: null,
|
||||
delivery_to: null,
|
||||
delivery_account_id: null,
|
||||
message: null,
|
||||
continuation_json: null,
|
||||
doctor_hint: null,
|
||||
stats_json: null,
|
||||
payload_json: "{}",
|
||||
updated_at_ms: 1,
|
||||
}),
|
||||
);
|
||||
const expected = payload(456);
|
||||
await writeLegacy(stateDir, { version: 1, payload: expected });
|
||||
|
||||
const result = await migrate({ env, stateDir });
|
||||
|
||||
expect(result.changes).toEqual([
|
||||
"Replaced an invalid SQLite restart sentinel with validated legacy state.",
|
||||
]);
|
||||
await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: expected });
|
||||
});
|
||||
|
||||
it("records and removes malformed transient state without disclosing its contents", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const sourcePath = await writeLegacy(stateDir, {
|
||||
version: 1,
|
||||
payload: { ...payload(), ts: "invalid", message: "secret-marker" },
|
||||
});
|
||||
|
||||
const result = await migrate({ env, stateDir });
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.changes).toEqual([
|
||||
"Discarded malformed retired restart sentinel JSON without importing it.",
|
||||
]);
|
||||
expect(JSON.stringify(result)).not.toContain("secret-marker");
|
||||
expect(receipt(env)?.report_json).not.toContain("secret-marker");
|
||||
await expect(readRestartSentinel(env)).resolves.toBeNull();
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("treats a completed receipt as authoritative if the retired file reappears", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
await writeLegacy(stateDir, { version: 1, payload: payload(1) });
|
||||
await migrate({ env, stateDir });
|
||||
await clearRestartSentinel(env);
|
||||
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(2) });
|
||||
|
||||
const result = await migrate({ env, stateDir });
|
||||
|
||||
expect(result.changes).toEqual([
|
||||
"Discarded recreated retired restart sentinel JSON using its migration receipt.",
|
||||
]);
|
||||
await expect(readRestartSentinel(env)).resolves.toBeNull();
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("recovers an interrupted claim and finishes the same migration owner", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() });
|
||||
await fsp.rename(sourcePath, `${sourcePath}.doctor-importing`);
|
||||
|
||||
const result = await migrate({ env, stateDir });
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: payload() });
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves changed source bytes and records no receipt", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload(1) });
|
||||
|
||||
const result = await migrate({
|
||||
env,
|
||||
stateDir,
|
||||
beforeVerify: () => {
|
||||
fs.writeFileSync(sourcePath, JSON.stringify({ version: 1, payload: payload(2) }));
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warnings).toHaveLength(1);
|
||||
expect(result.warnings[0]).toContain("changed after migration loaded it");
|
||||
expect(fs.existsSync(sourcePath)).toBe(true);
|
||||
expect(receipt(env)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retains a claimed source after cleanup failure and converges on retry", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() });
|
||||
const first = await migrate({
|
||||
env,
|
||||
stateDir,
|
||||
removeSource: () => {
|
||||
throw new Error("forced cleanup failure");
|
||||
},
|
||||
});
|
||||
|
||||
expect(first.warnings).toHaveLength(1);
|
||||
expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(true);
|
||||
expect(receipt(env)).toMatchObject({ removed_source: 0 });
|
||||
|
||||
const second = await migrate({ env, stateDir });
|
||||
expect(second.warnings).toEqual([]);
|
||||
expect(second.changes).toEqual([
|
||||
"Discarded recreated retired restart sentinel JSON using its migration receipt.",
|
||||
]);
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false);
|
||||
expect(receipt(env)).toMatchObject({ removed_source: 1 });
|
||||
});
|
||||
|
||||
it("requires exclusive state ownership before claiming the retired file", async () => {
|
||||
const { env, stateDir } = useStateDir();
|
||||
const sourcePath = await writeLegacy(stateDir, { version: 1, payload: payload() });
|
||||
const gatewayLock = await acquireGatewayLock({
|
||||
allowInTests: true,
|
||||
env,
|
||||
pollIntervalMs: 10,
|
||||
port: 18_791,
|
||||
timeoutMs: 100,
|
||||
});
|
||||
if (!gatewayLock) {
|
||||
throw new Error("expected test Gateway lock");
|
||||
}
|
||||
let result: Awaited<ReturnType<typeof migrateLegacyRestartSentinel>>;
|
||||
try {
|
||||
result = await migrate({ env, stateDir });
|
||||
} finally {
|
||||
await gatewayLock.release();
|
||||
}
|
||||
|
||||
expect(result.warnings[0]).toContain("Gateway or another SQLite maintenance command");
|
||||
expect(fs.existsSync(sourcePath)).toBe(true);
|
||||
expect(receipt(env)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects symlinks, hardlinks, and oversized sources without deleting them", async () => {
|
||||
const cases = ["symlink", "hardlink", "oversized"] as const;
|
||||
for (const sourceKind of cases) {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
const { env, stateDir } = useStateDir();
|
||||
const sourcePath = path.join(stateDir, "restart-sentinel.json");
|
||||
if (sourceKind === "oversized") {
|
||||
await fsp.writeFile(sourcePath, Buffer.alloc(4 * 1024 * 1024 + 1));
|
||||
} else {
|
||||
const targetPath = path.join(stateDir, `${sourceKind}-target.json`);
|
||||
await fsp.writeFile(targetPath, JSON.stringify({ version: 1, payload: payload() }));
|
||||
if (sourceKind === "symlink") {
|
||||
await fsp.symlink(targetPath, sourcePath);
|
||||
} else {
|
||||
await fsp.link(targetPath, sourcePath);
|
||||
}
|
||||
}
|
||||
|
||||
const result = await migrate({ env, stateDir });
|
||||
|
||||
expect(result.warnings).toHaveLength(1);
|
||||
expect(fs.existsSync(sourcePath)).toBe(true);
|
||||
expect(receipt(env)).toBeUndefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,536 @@
|
||||
// Startup/Doctor migration for the retired restart-sentinel JSON file.
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { root, type Root } from "@openclaw/fs-safe";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
runOpenClawStateWriteTransaction,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { formatErrorMessage } from "./errors.js";
|
||||
import { acquireGatewayLock, GatewayLockError } from "./gateway-lock.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "./kysely-sync.js";
|
||||
import {
|
||||
parseRestartSentinelEnvelope,
|
||||
readRestartSentinelRowSync,
|
||||
writeRestartSentinelRowSync,
|
||||
type RestartSentinelEnvelope,
|
||||
} from "./restart-sentinel-store.js";
|
||||
import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js";
|
||||
import type { MigrationMessages } from "./state-migrations.types.js";
|
||||
|
||||
const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json";
|
||||
const DOCTOR_CLAIM_SUFFIX = ".doctor-importing";
|
||||
const MAX_LEGACY_RESTART_SENTINEL_BYTES = 4 * 1024 * 1024;
|
||||
const MIGRATION_KIND = "legacy-restart-sentinel-json";
|
||||
const MIGRATION_LOCK_TIMEOUT_MS = 250;
|
||||
const MIGRATION_LOCK_POLL_INTERVAL_MS = 25;
|
||||
const utf8Decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
|
||||
type RestartSentinelMigrationDatabase = Pick<
|
||||
OpenClawStateKyselyDatabase,
|
||||
"gateway_restart_sentinel" | "migration_runs" | "migration_sources"
|
||||
>;
|
||||
|
||||
type LegacySourceSnapshot = {
|
||||
buffer: Buffer;
|
||||
dev: number;
|
||||
ino: number;
|
||||
mtimeMs: number;
|
||||
sha256: string;
|
||||
size: number;
|
||||
};
|
||||
|
||||
type MigrationDecision =
|
||||
| "canonical-preserved"
|
||||
| "invalid-canonical-repaired"
|
||||
| "legacy-imported"
|
||||
| "malformed-legacy-discarded"
|
||||
| "receipt-authoritative";
|
||||
|
||||
function legacyPathMayExist(filePath: string): boolean {
|
||||
try {
|
||||
fs.lstatSync(filePath);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return (error as NodeJS.ErrnoException).code !== "ENOENT";
|
||||
}
|
||||
}
|
||||
|
||||
/** Detect the exact retired file for startup preflight and explicit Doctor alike. */
|
||||
export function detectLegacyRestartSentinel(params: {
|
||||
stateDir: string;
|
||||
}): LegacyRestartSentinelDetection {
|
||||
const sourcePath = path.join(params.stateDir, LEGACY_RESTART_SENTINEL_FILENAME);
|
||||
return {
|
||||
sourcePath,
|
||||
hasLegacy:
|
||||
legacyPathMayExist(sourcePath) || legacyPathMayExist(`${sourcePath}${DOCTOR_CLAIM_SUFFIX}`),
|
||||
};
|
||||
}
|
||||
|
||||
function relativeLegacyPath(stateDir: string, filePath: string): string {
|
||||
const relativePath = path.relative(path.resolve(stateDir), path.resolve(filePath));
|
||||
if (
|
||||
!relativePath ||
|
||||
relativePath === ".." ||
|
||||
relativePath.startsWith(`..${path.sep}`) ||
|
||||
path.isAbsolute(relativePath)
|
||||
) {
|
||||
throw new Error("legacy restart sentinel path is outside the state directory");
|
||||
}
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
async function readLegacySourceSnapshot(
|
||||
stateRoot: Root,
|
||||
stateDir: string,
|
||||
sourcePath: string,
|
||||
): Promise<LegacySourceSnapshot> {
|
||||
const opened = await stateRoot.read(relativeLegacyPath(stateDir, sourcePath), {
|
||||
hardlinks: "reject",
|
||||
maxBytes: MAX_LEGACY_RESTART_SENTINEL_BYTES,
|
||||
symlinks: "reject",
|
||||
});
|
||||
if (!opened.stat.isFile() || opened.stat.size !== opened.buffer.byteLength) {
|
||||
throw new Error("legacy restart sentinel is not a stable regular file");
|
||||
}
|
||||
return {
|
||||
buffer: opened.buffer,
|
||||
dev: opened.stat.dev,
|
||||
ino: opened.stat.ino,
|
||||
mtimeMs: opened.stat.mtimeMs,
|
||||
sha256: createHash("sha256").update(opened.buffer).digest("hex"),
|
||||
size: opened.stat.size,
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotsMatch(left: LegacySourceSnapshot, right: LegacySourceSnapshot): boolean {
|
||||
return (
|
||||
left.dev === right.dev &&
|
||||
left.ino === right.ino &&
|
||||
left.mtimeMs === right.mtimeMs &&
|
||||
left.sha256 === right.sha256 &&
|
||||
left.size === right.size
|
||||
);
|
||||
}
|
||||
|
||||
function parseLegacyEnvelope(snapshot: LegacySourceSnapshot): RestartSentinelEnvelope | null {
|
||||
try {
|
||||
return parseRestartSentinelEnvelope(JSON.parse(utf8Decoder.decode(snapshot.buffer)));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function receiptSourceKey(sourcePath: string): string {
|
||||
return `restart-sentinel-json:${createHash("sha256").update(path.resolve(sourcePath)).digest("hex")}`;
|
||||
}
|
||||
|
||||
function hasMigrationReceipt(sourcePath: string, env: NodeJS.ProcessEnv): boolean {
|
||||
const { db } = openOpenClawStateDatabase({ env });
|
||||
return Boolean(
|
||||
executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
getNodeSqliteKysely<RestartSentinelMigrationDatabase>(db)
|
||||
.selectFrom("migration_sources")
|
||||
.select("source_key")
|
||||
.where("source_key", "=", receiptSourceKey(sourcePath)),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function decideAndRecordMigration(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
sourcePath: string;
|
||||
snapshot: LegacySourceSnapshot;
|
||||
envelope: RestartSentinelEnvelope | null;
|
||||
}): { decision: MigrationDecision; sourceKey: string } {
|
||||
const sourceKey = receiptSourceKey(params.sourcePath);
|
||||
const runId = `${sourceKey}:${params.snapshot.sha256.slice(0, 16)}`;
|
||||
const now = Date.now();
|
||||
return runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
const stateDb = getNodeSqliteKysely<RestartSentinelMigrationDatabase>(db);
|
||||
const receipt = executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("migration_sources")
|
||||
.select("source_key")
|
||||
.where("source_key", "=", sourceKey),
|
||||
);
|
||||
const before = readRestartSentinelRowSync(db);
|
||||
let decision: MigrationDecision;
|
||||
if (receipt) {
|
||||
decision = "receipt-authoritative";
|
||||
} else if (!params.envelope) {
|
||||
decision = "malformed-legacy-discarded";
|
||||
} else if (before.kind === "valid") {
|
||||
decision = "canonical-preserved";
|
||||
} else {
|
||||
const written = writeRestartSentinelRowSync(db, params.envelope.payload);
|
||||
const verified = readRestartSentinelRowSync(db);
|
||||
if (
|
||||
verified.kind !== "valid" ||
|
||||
verified.sentinel.revision !== written.revision ||
|
||||
!isDeepStrictEqual(verified.sentinel.payload, params.envelope.payload)
|
||||
) {
|
||||
throw new Error("SQLite verification failed for the restart sentinel migration");
|
||||
}
|
||||
decision = before.kind === "invalid" ? "invalid-canonical-repaired" : "legacy-imported";
|
||||
}
|
||||
|
||||
const reportJson = JSON.stringify({
|
||||
source: MIGRATION_KIND,
|
||||
target: "gateway_restart_sentinel",
|
||||
decision,
|
||||
sourceSha256: params.snapshot.sha256,
|
||||
sourceValid: params.envelope !== null,
|
||||
importedRecordCount:
|
||||
decision === "legacy-imported" || decision === "invalid-canonical-repaired" ? 1 : 0,
|
||||
preservedSqliteRecordCount: decision === "canonical-preserved" ? 1 : 0,
|
||||
});
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.insertInto("migration_runs")
|
||||
.values({
|
||||
id: runId,
|
||||
started_at: now,
|
||||
finished_at: now,
|
||||
status: "completed",
|
||||
report_json: reportJson,
|
||||
})
|
||||
.onConflict((conflict) =>
|
||||
conflict.column("id").doUpdateSet({
|
||||
finished_at: now,
|
||||
status: "completed",
|
||||
report_json: reportJson,
|
||||
}),
|
||||
),
|
||||
);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.insertInto("migration_sources")
|
||||
.values({
|
||||
source_key: sourceKey,
|
||||
migration_kind: MIGRATION_KIND,
|
||||
source_path: params.sourcePath,
|
||||
target_table: "gateway_restart_sentinel",
|
||||
source_sha256: params.snapshot.sha256,
|
||||
source_size_bytes: params.snapshot.size,
|
||||
source_record_count: params.envelope ? 1 : 0,
|
||||
last_run_id: runId,
|
||||
status: "completed",
|
||||
imported_at: now,
|
||||
removed_source: 0,
|
||||
report_json: reportJson,
|
||||
})
|
||||
.onConflict((conflict) =>
|
||||
conflict.column("source_key").doUpdateSet({
|
||||
source_sha256: params.snapshot.sha256,
|
||||
source_size_bytes: params.snapshot.size,
|
||||
source_record_count: params.envelope ? 1 : 0,
|
||||
last_run_id: runId,
|
||||
status: "completed",
|
||||
imported_at: now,
|
||||
removed_source: 0,
|
||||
report_json: reportJson,
|
||||
}),
|
||||
),
|
||||
);
|
||||
return { decision, sourceKey };
|
||||
},
|
||||
{ env: params.env },
|
||||
);
|
||||
}
|
||||
|
||||
function markSourceRemoved(sourceKey: string, env: NodeJS.ProcessEnv): void {
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
getNodeSqliteKysely<RestartSentinelMigrationDatabase>(db)
|
||||
.updateTable("migration_sources")
|
||||
.set({ removed_source: 1 })
|
||||
.where("source_key", "=", sourceKey),
|
||||
);
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
}
|
||||
|
||||
async function restoreClaim(params: {
|
||||
stateRoot: Root;
|
||||
stateDir: string;
|
||||
sourcePath: string;
|
||||
}): Promise<string | null> {
|
||||
const claimPath = `${params.sourcePath}${DOCTOR_CLAIM_SUFFIX}`;
|
||||
try {
|
||||
if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, claimPath)))) {
|
||||
return null;
|
||||
}
|
||||
if (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.sourcePath))) {
|
||||
return `source path already exists: ${params.sourcePath}`;
|
||||
}
|
||||
await params.stateRoot.move(
|
||||
relativeLegacyPath(params.stateDir, claimPath),
|
||||
relativeLegacyPath(params.stateDir, params.sourcePath),
|
||||
);
|
||||
return null;
|
||||
} catch (error) {
|
||||
return String(error);
|
||||
}
|
||||
}
|
||||
|
||||
async function recoverInterruptedClaim(params: {
|
||||
stateRoot: Root;
|
||||
stateDir: string;
|
||||
sourcePath: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): Promise<void> {
|
||||
const claimPath = `${params.sourcePath}${DOCTOR_CLAIM_SUFFIX}`;
|
||||
const claimRelativePath = relativeLegacyPath(params.stateDir, claimPath);
|
||||
if (!(await params.stateRoot.exists(claimRelativePath))) {
|
||||
return;
|
||||
}
|
||||
if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, params.sourcePath)))) {
|
||||
await params.stateRoot.move(
|
||||
claimRelativePath,
|
||||
relativeLegacyPath(params.stateDir, params.sourcePath),
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Both paths can only be retired safely when the claimed bytes already have
|
||||
// an authoritative decision; otherwise preserve both for operator recovery.
|
||||
if (!hasMigrationReceipt(params.sourcePath, params.env)) {
|
||||
throw new Error("legacy restart sentinel source and interrupted claim both exist");
|
||||
}
|
||||
await readLegacySourceSnapshot(params.stateRoot, params.stateDir, claimPath);
|
||||
await params.stateRoot.remove(claimRelativePath);
|
||||
}
|
||||
|
||||
function decisionChange(decision: MigrationDecision): string {
|
||||
switch (decision) {
|
||||
case "legacy-imported":
|
||||
return "Imported the legacy restart sentinel into shared SQLite state.";
|
||||
case "invalid-canonical-repaired":
|
||||
return "Replaced an invalid SQLite restart sentinel with validated legacy state.";
|
||||
case "canonical-preserved":
|
||||
return "Preserved the canonical SQLite restart sentinel and discarded conflicting legacy JSON.";
|
||||
case "malformed-legacy-discarded":
|
||||
return "Discarded malformed retired restart sentinel JSON without importing it.";
|
||||
case "receipt-authoritative":
|
||||
return "Discarded recreated retired restart sentinel JSON using its migration receipt.";
|
||||
}
|
||||
const unreachable: never = decision;
|
||||
return unreachable;
|
||||
}
|
||||
|
||||
async function migrateWithExclusiveStateOwnership(params: {
|
||||
detected: LegacyRestartSentinelDetection;
|
||||
stateRoot: Root;
|
||||
stateDir: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
beforeClaim?: () => void;
|
||||
beforeVerify?: () => void;
|
||||
removeSource?: (sourcePath: string) => Promise<void> | void;
|
||||
}): Promise<MigrationMessages> {
|
||||
const changes: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
const notices: string[] = [];
|
||||
const sourcePath = params.detected.sourcePath;
|
||||
try {
|
||||
await recoverInterruptedClaim({
|
||||
stateRoot: params.stateRoot,
|
||||
stateDir: params.stateDir,
|
||||
sourcePath,
|
||||
env: params.env,
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
changes,
|
||||
warnings: [`Failed recovering a legacy restart sentinel Doctor claim: ${String(error)}`],
|
||||
};
|
||||
}
|
||||
if (!(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath)))) {
|
||||
return { changes, warnings };
|
||||
}
|
||||
|
||||
let snapshot: LegacySourceSnapshot;
|
||||
try {
|
||||
snapshot = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, sourcePath);
|
||||
} catch (error) {
|
||||
return {
|
||||
changes,
|
||||
warnings: [`Failed reading the legacy restart sentinel: ${String(error)}`],
|
||||
};
|
||||
}
|
||||
const envelope = parseLegacyEnvelope(snapshot);
|
||||
const claimPath = `${sourcePath}${DOCTOR_CLAIM_SUFFIX}`;
|
||||
try {
|
||||
params.beforeVerify?.();
|
||||
const current = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, sourcePath);
|
||||
if (!snapshotsMatch(current, snapshot)) {
|
||||
throw new Error("legacy restart sentinel changed after migration loaded it");
|
||||
}
|
||||
params.beforeClaim?.();
|
||||
await params.stateRoot.move(
|
||||
relativeLegacyPath(params.stateDir, sourcePath),
|
||||
relativeLegacyPath(params.stateDir, claimPath),
|
||||
);
|
||||
const claimed = await readLegacySourceSnapshot(params.stateRoot, params.stateDir, claimPath);
|
||||
if (!snapshotsMatch(claimed, snapshot)) {
|
||||
throw new Error("legacy restart sentinel changed before migration could claim it");
|
||||
}
|
||||
} catch (error) {
|
||||
const restoreError = await restoreClaim({
|
||||
stateRoot: params.stateRoot,
|
||||
stateDir: params.stateDir,
|
||||
sourcePath,
|
||||
});
|
||||
return {
|
||||
changes,
|
||||
warnings: [
|
||||
`Failed claiming the legacy restart sentinel: ${String(error)}${restoreError ? `; restore failure: ${restoreError}` : ""}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
let result: ReturnType<typeof decideAndRecordMigration>;
|
||||
try {
|
||||
result = decideAndRecordMigration({
|
||||
env: params.env,
|
||||
sourcePath,
|
||||
snapshot,
|
||||
envelope,
|
||||
});
|
||||
} catch (error) {
|
||||
const restoreError = await restoreClaim({
|
||||
stateRoot: params.stateRoot,
|
||||
stateDir: params.stateDir,
|
||||
sourcePath,
|
||||
});
|
||||
return {
|
||||
changes,
|
||||
warnings: [
|
||||
`Failed migrating the legacy restart sentinel: ${String(error)}${restoreError ? `; restore failure: ${restoreError}` : ""}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
if (await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath))) {
|
||||
throw new Error("legacy restart sentinel reappeared during migration cleanup");
|
||||
}
|
||||
if (params.removeSource) {
|
||||
await params.removeSource(claimPath);
|
||||
} else {
|
||||
await params.stateRoot.remove(relativeLegacyPath(params.stateDir, claimPath));
|
||||
}
|
||||
if (
|
||||
(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, sourcePath))) ||
|
||||
(await params.stateRoot.exists(relativeLegacyPath(params.stateDir, claimPath)))
|
||||
) {
|
||||
throw new Error("legacy restart sentinel remains after migration cleanup");
|
||||
}
|
||||
} catch (error) {
|
||||
warnings.push(`Legacy restart sentinel cleanup failed: ${String(error)}`);
|
||||
return { changes, warnings };
|
||||
}
|
||||
|
||||
try {
|
||||
markSourceRemoved(result.sourceKey, params.env);
|
||||
} catch (error) {
|
||||
warnings.push(
|
||||
`Legacy restart sentinel was removed, but its receipt could not be finalized: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
changes.push(decisionChange(result.decision));
|
||||
notices.push("Removed retired restart-sentinel.json after recording its migration decision.");
|
||||
return { changes, warnings, notices };
|
||||
}
|
||||
|
||||
/** Import or retire the old file under exclusive state ownership. */
|
||||
export async function migrateLegacyRestartSentinel(params: {
|
||||
detected?: LegacyRestartSentinelDetection;
|
||||
stateDir: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
beforeClaim?: () => void;
|
||||
beforeVerify?: () => void;
|
||||
removeSource?: (sourcePath: string) => Promise<void> | void;
|
||||
}): Promise<MigrationMessages> {
|
||||
const detected = params.detected;
|
||||
if (!detected?.hasLegacy) {
|
||||
return { changes: [], warnings: [] };
|
||||
}
|
||||
const env = { ...(params.env ?? process.env), OPENCLAW_STATE_DIR: params.stateDir };
|
||||
let lock: Awaited<ReturnType<typeof acquireGatewayLock>>;
|
||||
try {
|
||||
lock = await acquireGatewayLock({
|
||||
allowInTests: true,
|
||||
env,
|
||||
pollIntervalMs: MIGRATION_LOCK_POLL_INTERVAL_MS,
|
||||
role: "sqlite-maintenance",
|
||||
timeoutMs: MIGRATION_LOCK_TIMEOUT_MS,
|
||||
});
|
||||
} catch (error) {
|
||||
const detail =
|
||||
error instanceof GatewayLockError
|
||||
? "the Gateway or another SQLite maintenance command owns this state directory"
|
||||
: String(error);
|
||||
return {
|
||||
changes: [],
|
||||
warnings: [
|
||||
`Failed migrating the legacy restart sentinel: ${detail}. Stop the Gateway, then run \`openclaw doctor --fix\` again.`,
|
||||
],
|
||||
};
|
||||
}
|
||||
if (!lock) {
|
||||
return {
|
||||
changes: [],
|
||||
warnings: [
|
||||
"Failed migrating the legacy restart sentinel: exclusive state ownership unavailable.",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
let result: MigrationMessages = { changes: [], warnings: [] };
|
||||
let releaseError: unknown;
|
||||
try {
|
||||
try {
|
||||
const stateRoot = await root(params.stateDir, {
|
||||
hardlinks: "reject",
|
||||
maxBytes: MAX_LEGACY_RESTART_SENTINEL_BYTES,
|
||||
symlinks: "reject",
|
||||
});
|
||||
result = await migrateWithExclusiveStateOwnership({
|
||||
...params,
|
||||
detected,
|
||||
env,
|
||||
stateRoot,
|
||||
});
|
||||
} catch (error) {
|
||||
result.warnings.push(`Failed reading the legacy restart sentinel: ${String(error)}`);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await lock.release();
|
||||
} catch (error) {
|
||||
releaseError = error;
|
||||
}
|
||||
}
|
||||
if (releaseError) {
|
||||
result.warnings.push(
|
||||
`Restart sentinel migration lock release failed: ${formatErrorMessage(releaseError)}`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export type LegacyRestartSentinelDetection = {
|
||||
sourcePath: string;
|
||||
hasLegacy: boolean;
|
||||
};
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
listWebPushSubscriptions,
|
||||
readPersistedVapidKeyPair,
|
||||
} from "./push-web-store.js";
|
||||
import { readRestartSentinel } from "./restart-sentinel.js";
|
||||
import {
|
||||
autoMigrateLegacyState,
|
||||
autoMigrateLegacyPluginDoctorState,
|
||||
@@ -1729,7 +1730,7 @@ describe("state migrations", () => {
|
||||
await expect(fs.readFile(`${routingPath}.migrated`, "utf8")).resolves.toContain("robot wake");
|
||||
});
|
||||
|
||||
it("auto-migrates standalone legacy voice wake JSON settings", async () => {
|
||||
it("auto-migrates standalone legacy JSON settings", async () => {
|
||||
const root = await createTempDir();
|
||||
const stateDir = path.join(root, ".openclaw");
|
||||
const env = createEnv(stateDir);
|
||||
@@ -1741,14 +1742,31 @@ describe("state migrations", () => {
|
||||
JSON.stringify({ triggers: ["wake"] }),
|
||||
"utf8",
|
||||
);
|
||||
const expectedSentinel = {
|
||||
kind: "update" as const,
|
||||
status: "ok" as const,
|
||||
ts: 321,
|
||||
message: "Update completed",
|
||||
};
|
||||
const restartSentinelPath = path.join(stateDir, "restart-sentinel.json");
|
||||
await fs.writeFile(
|
||||
restartSentinelPath,
|
||||
`${JSON.stringify({ version: 1, payload: expectedSentinel })}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await autoMigrateLegacyState({ cfg, env, homedir: () => root });
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
expect(result.migrated).toBe(true);
|
||||
expect(result.warnings).toStrictEqual([]);
|
||||
expect(result.changes).toContain(
|
||||
"Imported the legacy restart sentinel into shared SQLite state.",
|
||||
);
|
||||
await expect(loadVoiceWakeConfig(stateDir)).resolves.toMatchObject({ triggers: ["wake"] });
|
||||
await expect(readRestartSentinel(env)).resolves.toMatchObject({ payload: expectedSentinel });
|
||||
await expectMissingPath(path.join(settingsDir, "voicewake.json"));
|
||||
await expectMissingPath(restartSentinelPath);
|
||||
});
|
||||
|
||||
it("runs plugin doctor migrations after repairing shared state schema", async () => {
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { SessionScope } from "../config/sessions/types.js";
|
||||
import type { PluginDoctorStateMigration } from "../plugins/doctor-contract-registry.js";
|
||||
import type { LegacyChannelPairingStateDetection } from "./state-migrations.channel-pairing.js";
|
||||
import type { LegacyMcpOAuthDetection } from "./state-migrations.mcp-oauth.types.js";
|
||||
import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js";
|
||||
import type { LegacyWorkspaceStateDetection } from "./state-migrations.workspace-setup.types.js";
|
||||
|
||||
export type LegacyRescuePendingDetection = {
|
||||
@@ -115,6 +116,7 @@ export type LegacyStateDetection = {
|
||||
hasLegacy: boolean;
|
||||
};
|
||||
mcpOauth: LegacyMcpOAuthDetection;
|
||||
restartSentinel?: LegacyRestartSentinelDetection;
|
||||
workspace: LegacyWorkspaceStateDetection;
|
||||
webPush: {
|
||||
subscriptionsPath: string;
|
||||
|
||||
@@ -113,10 +113,14 @@ function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): voi
|
||||
doctorHint?: unknown;
|
||||
stats?: unknown;
|
||||
};
|
||||
const revision =
|
||||
typeof (sentinel as { revision?: unknown }).revision === "number"
|
||||
? (sentinel as { revision: number }).revision
|
||||
: Date.now();
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb.insertInto("gateway_restart_sentinel").values({
|
||||
sentinel_key: "current",
|
||||
sentinel_key: record.kind === "revision-floor" ? "revision-floor" : "current",
|
||||
version: 1,
|
||||
kind: typeof record.kind === "string" ? record.kind : "update",
|
||||
status: typeof record.status === "string" ? record.status : "skipped",
|
||||
@@ -136,22 +140,34 @@ function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): voi
|
||||
doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null,
|
||||
stats_json: record.stats ? JSON.stringify(record.stats) : null,
|
||||
payload_json: JSON.stringify(payload),
|
||||
updated_at_ms: Date.now(),
|
||||
updated_at_ms: revision,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function readRestartSentinelPayload(env: NodeJS.ProcessEnv): unknown {
|
||||
function replaceRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void {
|
||||
const { db } = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "current"),
|
||||
);
|
||||
writeRestartSentinelRow(env, sentinel);
|
||||
}
|
||||
|
||||
function readRestartSentinelPayload(env: NodeJS.ProcessEnv, key = "current"): unknown {
|
||||
const { db } = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select(["version", "payload_json"])
|
||||
.where("sentinel_key", "=", "current"),
|
||||
.select(["version", "payload_json", "updated_at_ms"])
|
||||
.where("sentinel_key", "=", key),
|
||||
);
|
||||
return row ? { version: row.version, payload: JSON.parse(row.payload_json) } : null;
|
||||
return row
|
||||
? { version: row.version, payload: JSON.parse(row.payload_json), revision: row.updated_at_ms }
|
||||
: null;
|
||||
}
|
||||
|
||||
async function runHelperWithExistingSentinel(params: {
|
||||
@@ -159,6 +175,8 @@ async function runHelperWithExistingSentinel(params: {
|
||||
metaHandoffId?: string;
|
||||
prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise<void> | void;
|
||||
sentinel?: unknown;
|
||||
parentExitTimeoutMs?: number;
|
||||
whileHelperRunning?: (env: NodeJS.ProcessEnv) => Promise<void> | void;
|
||||
}) {
|
||||
const { execFile } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
@@ -206,7 +224,7 @@ async function runHelperWithExistingSentinel(params: {
|
||||
{
|
||||
...helperParams,
|
||||
parentPid: process.pid,
|
||||
parentExitTimeoutMs: 1,
|
||||
parentExitTimeoutMs: params.parentExitTimeoutMs ?? 1,
|
||||
stateDatabasePath: resolveOpenClawStateSqlitePath(env),
|
||||
logPath: path.join(tmpDir, "handoff.log"),
|
||||
sensitivePaths: [],
|
||||
@@ -216,7 +234,7 @@ async function runHelperWithExistingSentinel(params: {
|
||||
)}\n`,
|
||||
);
|
||||
|
||||
const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
|
||||
const resultPromise = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
|
||||
(resolve) => {
|
||||
execFile(process.execPath, [helperScriptPath, helperParamsPath], (err) => {
|
||||
const childError = err as (NodeJS.ErrnoException & { signal?: NodeJS.Signals }) | null;
|
||||
@@ -227,6 +245,8 @@ async function runHelperWithExistingSentinel(params: {
|
||||
});
|
||||
},
|
||||
);
|
||||
await params.whileHelperRunning?.(env);
|
||||
const result = await resultPromise;
|
||||
|
||||
return { result, env };
|
||||
}
|
||||
@@ -868,7 +888,7 @@ describe("managed service update handoff", () => {
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toEqual(unrelatedSentinel);
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject(unrelatedSentinel);
|
||||
});
|
||||
|
||||
it("does not overwrite a newer pending update handoff sentinel", async () => {
|
||||
@@ -894,7 +914,74 @@ describe("managed service update handoff", () => {
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toEqual(newerSentinel);
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject(newerSentinel);
|
||||
});
|
||||
|
||||
it("preserves a newer sentinel written while the detached helper is active", async () => {
|
||||
const oldSentinel = {
|
||||
version: 1,
|
||||
revision: 100,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "skipped",
|
||||
ts: 100,
|
||||
stats: {
|
||||
handoffId: "old-handoff",
|
||||
reason: "managed-service-handoff-started",
|
||||
},
|
||||
},
|
||||
};
|
||||
const newerSentinel = {
|
||||
version: 1,
|
||||
revision: 200,
|
||||
payload: {
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 200,
|
||||
},
|
||||
};
|
||||
const { env } = await runHelperWithExistingSentinel({
|
||||
handoffId: "old-handoff",
|
||||
metaHandoffId: "old-handoff",
|
||||
sentinel: oldSentinel,
|
||||
parentExitTimeoutMs: 200,
|
||||
whileHelperRunning: async (stateEnv) => {
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, 50);
|
||||
});
|
||||
replaceRestartSentinelRow(stateEnv, newerSentinel);
|
||||
},
|
||||
});
|
||||
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
payload: newerSentinel.payload,
|
||||
revision: 200,
|
||||
});
|
||||
});
|
||||
|
||||
it("advances the durable revision floor even when it is ahead of the clock", async () => {
|
||||
const futureRevision = Date.now() + 60_000;
|
||||
const { env } = await runHelperWithExistingSentinel({
|
||||
handoffId: "handoff-future-revision",
|
||||
metaHandoffId: "handoff-future-revision",
|
||||
sentinel: {
|
||||
version: 1,
|
||||
revision: futureRevision,
|
||||
payload: {
|
||||
kind: "revision-floor",
|
||||
status: "skipped",
|
||||
ts: 123,
|
||||
stats: {
|
||||
handoffId: "handoff-future-revision",
|
||||
reason: "managed-service-handoff-started",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(readRestartSentinelPayload(env, "revision-floor")).toMatchObject({
|
||||
revision: futureRevision + 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("sweeps stale handoff temp directories while keeping fresh handoff logs", async () => {
|
||||
|
||||
@@ -201,84 +201,140 @@ function hardenStateDatabaseFiles() {
|
||||
}
|
||||
}
|
||||
|
||||
function readRestartSentinelPayload() {
|
||||
const db = openStateDatabase();
|
||||
if (!db) {
|
||||
function parseJsonColumn(value) {
|
||||
if (typeof value !== "string" || !value) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const row = db
|
||||
.prepare("SELECT version, payload_json FROM gateway_restart_sentinel WHERE sentinel_key = ?")
|
||||
.get("current");
|
||||
if (!row || row.version !== 1 || typeof row.payload_json !== "string") {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(row.payload_json);
|
||||
return JSON.parse(value);
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
hardenStateDatabaseFiles();
|
||||
try {
|
||||
db.close();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
function writeRestartSentinelPayload(payload) {
|
||||
const db = openStateDatabase();
|
||||
if (!db) {
|
||||
return;
|
||||
function readRestartSentinelRecord(db) {
|
||||
const row = db
|
||||
.prepare(
|
||||
[
|
||||
"SELECT version, kind, status, ts, session_key, thread_id,",
|
||||
"delivery_channel, delivery_to, delivery_account_id, message, continuation_json,",
|
||||
"doctor_hint, stats_json, updated_at_ms",
|
||||
"FROM gateway_restart_sentinel WHERE sentinel_key = ?",
|
||||
].join(" "),
|
||||
)
|
||||
.get("current");
|
||||
if (
|
||||
!row ||
|
||||
row.version !== 1 ||
|
||||
typeof row.kind !== "string" ||
|
||||
typeof row.status !== "string" ||
|
||||
typeof row.ts !== "number" ||
|
||||
typeof row.updated_at_ms !== "number"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const updatedAtMs = Date.now();
|
||||
db.prepare(
|
||||
const payload = {
|
||||
kind: row.kind,
|
||||
status: row.status,
|
||||
ts: row.ts,
|
||||
};
|
||||
if (typeof row.session_key === "string") payload.sessionKey = row.session_key;
|
||||
if (typeof row.thread_id === "string") payload.threadId = row.thread_id;
|
||||
const deliveryContext = {};
|
||||
if (typeof row.delivery_channel === "string") deliveryContext.channel = row.delivery_channel;
|
||||
if (typeof row.delivery_to === "string") deliveryContext.to = row.delivery_to;
|
||||
if (typeof row.delivery_account_id === "string") deliveryContext.accountId = row.delivery_account_id;
|
||||
if (Object.keys(deliveryContext).length > 0) payload.deliveryContext = deliveryContext;
|
||||
if (typeof row.message === "string") payload.message = row.message;
|
||||
const continuation = parseJsonColumn(row.continuation_json);
|
||||
if (continuation) payload.continuation = continuation;
|
||||
if (typeof row.doctor_hint === "string") payload.doctorHint = row.doctor_hint;
|
||||
const stats = parseJsonColumn(row.stats_json);
|
||||
if (stats) payload.stats = stats;
|
||||
return { revision: row.updated_at_ms, payload };
|
||||
}
|
||||
|
||||
function readRestartSentinelRevisionFloor(db) {
|
||||
const row = db
|
||||
.prepare("SELECT updated_at_ms FROM gateway_restart_sentinel WHERE sentinel_key = ?")
|
||||
.get("revision-floor");
|
||||
if (!row) return null;
|
||||
if (!Number.isSafeInteger(row.updated_at_ms)) {
|
||||
throw new Error("restart sentinel revision floor is outside the safe integer range");
|
||||
}
|
||||
return row.updated_at_ms;
|
||||
}
|
||||
|
||||
function advanceRestartSentinelRevisionFloor(db, revision) {
|
||||
const payloadJson = JSON.stringify({ kind: "restart", status: "skipped", ts: revision });
|
||||
db.prepare(
|
||||
[
|
||||
"INSERT INTO gateway_restart_sentinel (",
|
||||
"sentinel_key, version, kind, status, ts, session_key, thread_id,",
|
||||
"delivery_channel, delivery_to, delivery_account_id, message, continuation_json,",
|
||||
"doctor_hint, stats_json, payload_json, updated_at_ms",
|
||||
") VALUES ('revision-floor', 1, 'restart', 'skipped', ?, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ?, ?)",
|
||||
"ON CONFLICT(sentinel_key) DO UPDATE SET",
|
||||
"ts = excluded.ts, payload_json = excluded.payload_json, updated_at_ms = excluded.updated_at_ms",
|
||||
].join(" "),
|
||||
).run(revision, payloadJson, revision);
|
||||
}
|
||||
|
||||
function writeRestartSentinelPayload(db, payload, currentRevision) {
|
||||
const revisionFloor = readRestartSentinelRevisionFloor(db);
|
||||
const updatedAtMs = Math.max(Date.now(), Math.max(currentRevision || 0, revisionFloor || 0) + 1);
|
||||
if (!Number.isSafeInteger(updatedAtMs)) {
|
||||
throw new Error("restart sentinel revision exhausted the safe integer range");
|
||||
}
|
||||
const values = [
|
||||
payload.kind,
|
||||
payload.status,
|
||||
payload.ts,
|
||||
payload.sessionKey || null,
|
||||
payload.threadId || null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.channel === "string"
|
||||
? payload.deliveryContext.channel
|
||||
: null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.to === "string"
|
||||
? payload.deliveryContext.to
|
||||
: null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.accountId === "string"
|
||||
? payload.deliveryContext.accountId
|
||||
: null,
|
||||
payload.message || null,
|
||||
payload.continuation ? JSON.stringify(payload.continuation) : null,
|
||||
payload.doctorHint || null,
|
||||
payload.stats ? JSON.stringify(payload.stats) : null,
|
||||
JSON.stringify(payload),
|
||||
updatedAtMs,
|
||||
];
|
||||
let changed;
|
||||
if (currentRevision === null) {
|
||||
changed = db.prepare(
|
||||
[
|
||||
"INSERT INTO gateway_restart_sentinel (",
|
||||
"sentinel_key, version, kind, status, ts, session_key, thread_id,",
|
||||
"delivery_channel, delivery_to, delivery_account_id, message, continuation_json,",
|
||||
"doctor_hint, stats_json, payload_json, updated_at_ms",
|
||||
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
"ON CONFLICT(sentinel_key) DO UPDATE SET",
|
||||
"version = excluded.version, kind = excluded.kind, status = excluded.status,",
|
||||
"ts = excluded.ts, session_key = excluded.session_key, thread_id = excluded.thread_id,",
|
||||
"delivery_channel = excluded.delivery_channel, delivery_to = excluded.delivery_to,",
|
||||
"delivery_account_id = excluded.delivery_account_id, message = excluded.message,",
|
||||
"continuation_json = excluded.continuation_json, doctor_hint = excluded.doctor_hint,",
|
||||
"stats_json = excluded.stats_json, payload_json = excluded.payload_json,",
|
||||
"updated_at_ms = excluded.updated_at_ms",
|
||||
") VALUES ('current', 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
].join(" "),
|
||||
).run(
|
||||
"current",
|
||||
1,
|
||||
payload.kind,
|
||||
payload.status,
|
||||
payload.ts,
|
||||
payload.sessionKey || null,
|
||||
payload.threadId || null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.channel === "string"
|
||||
? payload.deliveryContext.channel
|
||||
: null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.to === "string"
|
||||
? payload.deliveryContext.to
|
||||
: null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.accountId === "string"
|
||||
? payload.deliveryContext.accountId
|
||||
: null,
|
||||
payload.message || null,
|
||||
payload.continuation ? JSON.stringify(payload.continuation) : null,
|
||||
payload.doctorHint || null,
|
||||
payload.stats ? JSON.stringify(payload.stats) : null,
|
||||
JSON.stringify(payload),
|
||||
updatedAtMs,
|
||||
);
|
||||
} catch (err) {
|
||||
appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err)));
|
||||
} finally {
|
||||
hardenStateDatabaseFiles();
|
||||
try {
|
||||
db.close();
|
||||
} catch {}
|
||||
).run(...values).changes === 1;
|
||||
} else {
|
||||
changed = db.prepare(
|
||||
[
|
||||
"UPDATE gateway_restart_sentinel SET",
|
||||
"version = 1, kind = ?, status = ?, ts = ?, session_key = ?, thread_id = ?,",
|
||||
"delivery_channel = ?, delivery_to = ?, delivery_account_id = ?, message = ?,",
|
||||
"continuation_json = ?, doctor_hint = ?, stats_json = ?, payload_json = ?, updated_at_ms = ?",
|
||||
"WHERE sentinel_key = 'current' AND updated_at_ms = ?",
|
||||
].join(" "),
|
||||
).run(...values, currentRevision).changes === 1;
|
||||
}
|
||||
if (changed) {
|
||||
// This runs inside the same BEGIN IMMEDIATE section as the guarded current-row write.
|
||||
advanceRestartSentinelRevisionFloor(db, updatedAtMs);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
function buildFallbackFailurePayload(reason) {
|
||||
@@ -312,22 +368,77 @@ function buildFallbackFailurePayload(reason) {
|
||||
}
|
||||
|
||||
function markUpdateSentinelFailureIfPending(reason) {
|
||||
let payload = readRestartSentinelPayload();
|
||||
if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) {
|
||||
const snapshotDb = openStateDatabase();
|
||||
if (!snapshotDb) return;
|
||||
let snapshot;
|
||||
try {
|
||||
snapshot = readRestartSentinelRecord(snapshotDb);
|
||||
} catch {
|
||||
return;
|
||||
} finally {
|
||||
try {
|
||||
snapshotDb.close();
|
||||
} catch {}
|
||||
}
|
||||
const handoffId = typeof params.handoffId === "string" ? params.handoffId.trim() : "";
|
||||
if (payload && handoffId && (!payload.stats || payload.stats.handoffId !== handoffId)) {
|
||||
return;
|
||||
const fallbackPayload = snapshot === null ? buildFallbackFailurePayload(reason) : null;
|
||||
|
||||
const db = openStateDatabase();
|
||||
if (!db) return;
|
||||
let transactionOpen = false;
|
||||
try {
|
||||
db.exec("BEGIN IMMEDIATE;");
|
||||
transactionOpen = true;
|
||||
const current = readRestartSentinelRecord(db);
|
||||
if (
|
||||
(snapshot === null && current !== null) ||
|
||||
(snapshot !== null &&
|
||||
(current === null || current.revision !== snapshot.revision))
|
||||
) {
|
||||
db.exec("COMMIT;");
|
||||
transactionOpen = false;
|
||||
return;
|
||||
}
|
||||
|
||||
let payload = current && current.payload;
|
||||
if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) {
|
||||
db.exec("COMMIT;");
|
||||
transactionOpen = false;
|
||||
return;
|
||||
}
|
||||
const handoffId = typeof params.handoffId === "string" ? params.handoffId.trim() : "";
|
||||
if (payload && handoffId && (!payload.stats || payload.stats.handoffId !== handoffId)) {
|
||||
db.exec("COMMIT;");
|
||||
transactionOpen = false;
|
||||
return;
|
||||
}
|
||||
if (payload) {
|
||||
payload = { ...payload, status: "error" };
|
||||
delete payload.continuation;
|
||||
payload.stats = { ...(payload.stats || {}), reason };
|
||||
} else {
|
||||
payload = fallbackPayload;
|
||||
}
|
||||
if (!payload) {
|
||||
throw new Error("restart sentinel disappeared before guarded failure write");
|
||||
}
|
||||
if (!writeRestartSentinelPayload(db, payload, current ? current.revision : null)) {
|
||||
throw new Error("restart sentinel changed before guarded failure write");
|
||||
}
|
||||
db.exec("COMMIT;");
|
||||
transactionOpen = false;
|
||||
} catch (err) {
|
||||
if (transactionOpen) {
|
||||
try {
|
||||
db.exec("ROLLBACK;");
|
||||
} catch {}
|
||||
}
|
||||
appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err)));
|
||||
} finally {
|
||||
hardenStateDatabaseFiles();
|
||||
try {
|
||||
db.close();
|
||||
} catch {}
|
||||
}
|
||||
if (payload) {
|
||||
payload = { ...payload, status: "error" };
|
||||
delete payload.continuation;
|
||||
payload.stats = { ...(payload.stats || {}), reason };
|
||||
} else {
|
||||
payload = buildFallbackFailurePayload(reason);
|
||||
}
|
||||
writeRestartSentinelPayload(payload);
|
||||
}
|
||||
|
||||
function runServiceCommand(command, args) {
|
||||
|
||||
@@ -128,6 +128,84 @@ describe("check-database-first-legacy-stores", () => {
|
||||
expect(violations).toEqual([{ kind: "legacy store filesystem write", line: 5 }]);
|
||||
});
|
||||
|
||||
it("keeps legacy restart sentinel filesystem access in its sole migration owner", () => {
|
||||
const runtimeViolations = collectDatabaseFirstLegacyStoreViolations(
|
||||
`
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
const legacyFilename = "restart-sentinel.json";
|
||||
`,
|
||||
"src/infra/restart-sentinel.ts",
|
||||
);
|
||||
const migrationViolations = collectDatabaseFirstLegacyStoreViolations(
|
||||
`
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
const legacyFilename = "restart-sentinel.json";
|
||||
`,
|
||||
"src/infra/state-migrations.restart-sentinel.ts",
|
||||
);
|
||||
|
||||
expect(runtimeViolations).toEqual([
|
||||
{ kind: "legacy restart sentinel filesystem import", line: 2 },
|
||||
{ kind: "legacy restart sentinel filesystem import", line: 3 },
|
||||
{ kind: "legacy restart sentinel reference", line: 4 },
|
||||
]);
|
||||
expect(migrationViolations).toEqual([]);
|
||||
});
|
||||
|
||||
it("flags legacy restart sentinel references outside the migration owner", () => {
|
||||
const violations = collectDatabaseFirstLegacyStoreViolations(
|
||||
`
|
||||
const legacyPath = path.join(stateDir, "restart-sentinel.json");
|
||||
await readFile(legacyPath, "utf8");
|
||||
`,
|
||||
"src/commands/doctor/state-migrations.ts",
|
||||
);
|
||||
|
||||
expect(violations).toEqual([{ kind: "legacy restart sentinel reference", line: 2 }]);
|
||||
});
|
||||
|
||||
it("allows the CLI preflight to detect exact legacy restart sentinel inputs", () => {
|
||||
const violations = collectDatabaseFirstLegacyStoreViolations(
|
||||
`
|
||||
[
|
||||
path.join(stateDir, "restart-sentinel.json"),
|
||||
path.join(stateDir, "restart-sentinel.json.doctor-importing"),
|
||||
].some(fileOrDirExists);
|
||||
`,
|
||||
"src/cli/program/config-guard.ts",
|
||||
);
|
||||
|
||||
expect(violations).toEqual([]);
|
||||
});
|
||||
|
||||
it("flags direct legacy restart sentinel reads from the CLI preflight", () => {
|
||||
const violations = collectDatabaseFirstLegacyStoreViolations(
|
||||
`
|
||||
await readFile(path.join(stateDir, "restart-sentinel.json"), "utf8");
|
||||
await readFile(path.join(stateDir, "restart-sentinel.json.doctor-importing"), "utf8");
|
||||
`,
|
||||
"src/cli/program/config-guard.ts",
|
||||
);
|
||||
|
||||
expect(violations).toEqual([
|
||||
{ kind: "legacy restart sentinel reference", line: 2 },
|
||||
{ kind: "legacy restart sentinel reference", line: 3 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("flags nested restart sentinel paths disguised as CLI preflight detection", () => {
|
||||
const violations = collectDatabaseFirstLegacyStoreViolations(
|
||||
`
|
||||
[path.join(stateDir, "archive/restart-sentinel.json")].some(fileOrDirExists);
|
||||
`,
|
||||
"src/cli/program/config-guard.ts",
|
||||
);
|
||||
|
||||
expect(violations).toEqual([{ kind: "legacy restart sentinel reference", line: 2 }]);
|
||||
});
|
||||
|
||||
it("flags retired Diffs viewer sidecar writes", () => {
|
||||
const violations = collectDatabaseFirstLegacyStoreViolations(
|
||||
`
|
||||
@@ -1819,6 +1897,7 @@ describe("check-database-first-legacy-stores", () => {
|
||||
);
|
||||
|
||||
expect(violations).toEqual([
|
||||
{ kind: "legacy restart sentinel reference", line: 5 },
|
||||
{ kind: "legacy store filesystem write", line: 5 },
|
||||
{ kind: "legacy store filesystem write", line: 6 },
|
||||
{ kind: "legacy store filesystem write", line: 7 },
|
||||
|
||||
Reference in New Issue
Block a user