mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(cron): fence concurrent executions durably
This commit is contained in:
@@ -68,6 +68,7 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
### Fixes
|
||||
|
||||
- **Cron execution ownership:** record durable SQLite run receipts, fence overlapping gateway schedulers by exact process ownership, and supersede live results when the job's effective agent changes.
|
||||
- **Control UI session companion:** load bounded visible session context before answering, keep unavailable questions retryable, and prevent private companion reference wrappers from appearing as answers. Fixes #120746. Thanks @shakkernerd.
|
||||
- **Telegram live locations:** expose initial, moving, and stopped live-location updates through the channel-neutral `message_received` hook without starting agent turns for edits.
|
||||
- **Updater plugin convergence:** keep pre-plugin doctor passes from installing configured plugins before the updater's plugin sweep, while preserving the final post-plugin migration pass and preventing ambient update-phase state from leaking into fresh doctor processes.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { materializeLegacyDefaultCronJobOwners } from "../legacy-default-agent-owner-migration.js";
|
||||
import { reconcileCronRunReceiptForStartup } from "../store/run-receipt-store.js";
|
||||
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
import { nextWakeAtMs, recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
@@ -66,6 +67,18 @@ export async function start(state: CronServiceState) {
|
||||
// Older releases used runningAtMs for both queued and active work. Those
|
||||
// rows are intentionally recovered conservatively to avoid replaying side effects.
|
||||
const runningAtMs = job.state.runningAtMs;
|
||||
const liveReceipt = reconcileCronRunReceiptForStartup({
|
||||
storePath: state.deps.storePath,
|
||||
jobId: job.id,
|
||||
startedAtMs: runningAtMs,
|
||||
nowMs: state.deps.nowMs(),
|
||||
});
|
||||
if (liveReceipt) {
|
||||
// An overlapping replacement gateway must not retire work whose
|
||||
// exact process incarnation is still alive.
|
||||
interruptedJobIds.add(job.id);
|
||||
continue;
|
||||
}
|
||||
const taskRunId = tryFindCronTaskRunIdForRecovery(state, job.id, runningAtMs);
|
||||
const finalized = tryFindFinalizedCronTaskRun(state, job.id, runningAtMs);
|
||||
if (finalized) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { CommandLaneTaskMarker } from "../../process/command-queue.js";
|
||||
import type { CronActiveJobMarker } from "../active-jobs.js";
|
||||
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
|
||||
import type { CronRunReceiptHandle } from "../store/run-receipt-store.js";
|
||||
import type { CronJob, CronPayload, CronRunErrorClassification } from "../types.js";
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
@@ -71,6 +72,7 @@ export type ActivatedManualRun = Extract<PreparedManualRun, { ran: true }> & {
|
||||
activeJobMarker?: CronActiveJobMarker;
|
||||
admittedJob: CronJob;
|
||||
executionJob: CronJob;
|
||||
runReceipt: CronRunReceiptHandle;
|
||||
};
|
||||
|
||||
export type ManualRunOptions = {
|
||||
@@ -483,13 +485,17 @@ export async function activatePreparedManualRun(
|
||||
if (activation.kind === "unavailable") {
|
||||
return { ok: true, ran: false, reason: activation.reason } as const;
|
||||
}
|
||||
if (activation.kind === "fenced") {
|
||||
await releasePreparedManualReservationWithRetry(state, prepared);
|
||||
return { ok: true, ran: false, reason: "already-running" } as const;
|
||||
}
|
||||
const { startedAt } = activation;
|
||||
emit(state, { jobId: job.id, action: "started", job, runAtMs: startedAt });
|
||||
const taskRunId = tryCreateCronTaskRun({
|
||||
state,
|
||||
job,
|
||||
startedAt,
|
||||
publicRunId: prepared.runId,
|
||||
publicRunId: prepared.runId ?? activation.runReceipt.receiptId,
|
||||
});
|
||||
const activeJobMarker = markManualCronJobActive(state, job);
|
||||
// Execute against a snapshot so later reload/merge can preserve delivery
|
||||
@@ -512,6 +518,7 @@ export async function activatePreparedManualRun(
|
||||
activeJobMarker,
|
||||
admittedJob,
|
||||
executionJob,
|
||||
runReceipt: activation.runReceipt,
|
||||
} as const;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { enqueueCommandInLane } from "../../process/command-queue.js";
|
||||
import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gateway-work-admission.js";
|
||||
import { CommandLane } from "../../process/lanes.js";
|
||||
import { isCronActiveJobMarkerCurrent } from "../active-jobs.js";
|
||||
import { CronRunReceiptRevisionError, finishCronRunReceipt } from "../store/run-receipt-store.js";
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
@@ -19,6 +20,11 @@ import {
|
||||
} from "./ops-run-preparation.js";
|
||||
import { clearManualCronJobActive, maybeNotifyManualIsolatedSetupTimeout } from "./ops-shared.js";
|
||||
import { releaseQueuedCronRun, runWithCronAdmission } from "./run-admission.js";
|
||||
import {
|
||||
assertServiceCronRunReceiptCurrent,
|
||||
cronRunReceiptPersistHooks,
|
||||
supersedeServiceCronRunReceipt,
|
||||
} from "./run-receipts.js";
|
||||
import { mergeManualRunSnapshotAfterReload } from "./startup-run-repair.js";
|
||||
import type { CronServiceState, CronWakeMode, DeferredCronNotifications } from "./state.js";
|
||||
import { emit } from "./state.js";
|
||||
@@ -55,6 +61,7 @@ async function finishPreparedManualRun(
|
||||
const jobId = prepared.jobId;
|
||||
const taskRunId = prepared.taskRunId;
|
||||
const runId = prepared.runId;
|
||||
let finalized = false;
|
||||
|
||||
try {
|
||||
let coreResult: Awaited<ReturnType<typeof executeJobCoreWithTimeout>>;
|
||||
@@ -66,6 +73,7 @@ async function finishPreparedManualRun(
|
||||
streamBatch: prepared.streamBatch,
|
||||
streamScheduleKey: prepared.streamScheduleKey,
|
||||
streamSourceIdentity: prepared.streamSourceIdentity,
|
||||
runReceipt: prepared.runReceipt,
|
||||
});
|
||||
} catch (err) {
|
||||
coreResult = { status: "error", error: normalizeCronRunErrorText(err) };
|
||||
@@ -147,7 +155,6 @@ async function finishPreparedManualRun(
|
||||
return;
|
||||
}
|
||||
|
||||
let finalized = false;
|
||||
let notifySetupTimeout = coreResult.isolatedAgentSetupTimeout !== undefined;
|
||||
await locked(state, async () => {
|
||||
await ensureLoaded(state, { skipRecompute: true });
|
||||
@@ -162,6 +169,16 @@ async function finishPreparedManualRun(
|
||||
if (!job) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
assertServiceCronRunReceiptCurrent(state, prepared.runReceipt);
|
||||
} catch (error) {
|
||||
if (error instanceof CronRunReceiptRevisionError) {
|
||||
supersedeServiceCronRunReceipt(prepared.runReceipt, state.deps.nowMs(), error.message);
|
||||
notifySetupTimeout = false;
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const scheduleOwnership = resolveCronRunScheduleOwnership({
|
||||
admittedJob: prepared.admittedJob,
|
||||
@@ -305,9 +322,27 @@ async function finishPreparedManualRun(
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
await persistOrRestore(state, rollbackSnapshot, {
|
||||
postPersistNotifications,
|
||||
});
|
||||
try {
|
||||
await persistOrRestore(state, rollbackSnapshot, {
|
||||
postPersistNotifications,
|
||||
transactionHooks: cronRunReceiptPersistHooks({
|
||||
state,
|
||||
handle: prepared.runReceipt,
|
||||
terminal: {
|
||||
status: triggerSkipped ? "skipped" : coreResult.status,
|
||||
finishedAtMs: endedAt,
|
||||
error: coreResult.error,
|
||||
},
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof CronRunReceiptRevisionError) {
|
||||
supersedeServiceCronRunReceipt(prepared.runReceipt, state.deps.nowMs(), error.message);
|
||||
notifySetupTimeout = false;
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (removedJob) {
|
||||
pruneCronJobScratchAfterCommit(state, [removedJob.id]);
|
||||
emit(state, { jobId: removedJob.id, action: "removed", job: removedJob });
|
||||
@@ -336,6 +371,14 @@ async function finishPreparedManualRun(
|
||||
}
|
||||
emitMissingQueuedTerminal();
|
||||
} finally {
|
||||
if (!finalized) {
|
||||
finishCronRunReceipt({
|
||||
handle: prepared.runReceipt,
|
||||
status: "superseded",
|
||||
finishedAtMs: state.deps.nowMs(),
|
||||
error: "cron run result was not applied to the current job revision",
|
||||
});
|
||||
}
|
||||
releaseQueuedCronRun(state, prepared.jobId, prepared.reservationIdentity);
|
||||
clearManualCronJobActive(state, jobId, prepared.activeJobMarker);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
import { type ChildProcess, spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import fsPromises from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { openOpenClawStateDatabase } from "../../state/openclaw-state-db.js";
|
||||
import { resolveOpenClawStateDirForDatabasePath } from "../../state/openclaw-state-db.paths.js";
|
||||
import { CronService } from "../service.js";
|
||||
import { createCronStoreHarness } from "../service.test-harness.js";
|
||||
import { loadCronStore, saveCronStore } from "../store.js";
|
||||
import { listCronRunReceipts } from "../store/run-receipt-store.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
|
||||
const { makeStorePath } = createCronStoreHarness({ prefix: "cron-owner-hardening-" });
|
||||
const children = new Set<ChildProcess>();
|
||||
let scriptRoot = "";
|
||||
let runnerScript = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
scriptRoot = await fsPromises.mkdtemp(path.join(os.tmpdir(), "cron-owner-hardening-script-"));
|
||||
runnerScript = path.join(scriptRoot, "runner.mts");
|
||||
const serviceUrl = pathToFileURL(path.resolve("src/cron/service.ts")).href;
|
||||
await fsPromises.writeFile(
|
||||
runnerScript,
|
||||
`
|
||||
import fs from "node:fs";
|
||||
import { CronService } from ${JSON.stringify(serviceUrl)};
|
||||
const [storePath, jobId, mode, releasePath, outputPath] = process.argv.slice(2);
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const logger = { debug() {}, info() {}, warn() {}, error() {} };
|
||||
const cron = new CronService({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
log: logger,
|
||||
enqueueSystemEvent() {},
|
||||
requestHeartbeat() {},
|
||||
evaluateCronTrigger: async () => {
|
||||
process.stdout.write("trigger\\n");
|
||||
while (!fs.existsSync(releasePath)) await sleep(10);
|
||||
return { kind: "evaluated", fire: true };
|
||||
},
|
||||
runIsolatedAgentJob: async () => ({ status: "ok" }),
|
||||
runCommandJob: async () => {
|
||||
fs.appendFileSync(outputPath, process.pid + "\\n");
|
||||
process.stdout.write("started\\n");
|
||||
if (mode === "block") await new Promise(() => {});
|
||||
await sleep(150);
|
||||
return { status: "ok", summary: "done" };
|
||||
},
|
||||
});
|
||||
await cron.start();
|
||||
if (mode === "block") await cron.run(jobId, "force");
|
||||
if (mode === "due") await sleep(350);
|
||||
cron.stop();
|
||||
`,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const child of children) {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
}
|
||||
children.clear();
|
||||
});
|
||||
|
||||
function makeCommandJob(id: string, nextRunAtMs: number, trigger = false): CronJob {
|
||||
return {
|
||||
id,
|
||||
agentId: "alpha",
|
||||
name: id,
|
||||
enabled: true,
|
||||
createdAtMs: nextRunAtMs - 1,
|
||||
updatedAtMs: nextRunAtMs - 1,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: nextRunAtMs },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
...(trigger ? { trigger: { script: "return true" } } : {}),
|
||||
payload: { kind: "command", argv: ["true"] },
|
||||
state: { nextRunAtMs },
|
||||
};
|
||||
}
|
||||
|
||||
function spawnRunner(params: {
|
||||
storePath: string;
|
||||
jobId: string;
|
||||
mode: "block" | "trigger" | "due";
|
||||
releasePath: string;
|
||||
outputPath: string;
|
||||
}): ChildProcess {
|
||||
const stateDir = resolveOpenClawStateDirForDatabasePath(openOpenClawStateDatabase().path);
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
runnerScript,
|
||||
params.storePath,
|
||||
params.jobId,
|
||||
params.mode,
|
||||
params.releasePath,
|
||||
params.outputPath,
|
||||
],
|
||||
{
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
children.add(child);
|
||||
return child;
|
||||
}
|
||||
|
||||
async function waitForLine(child: ChildProcess, expected: string): Promise<void> {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout?.on("data", (chunk) => {
|
||||
stdout += String(chunk);
|
||||
});
|
||||
child.stderr?.on("data", (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
await vi.waitFor(
|
||||
() => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
throw new Error(`cron child exited before ${expected}: ${stderr || stdout}`);
|
||||
}
|
||||
expect(stdout.split("\n")).toContain(expected);
|
||||
},
|
||||
{ timeout: 10_000, interval: 20 },
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForExit(child: ChildProcess): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
child.once("exit", () => resolve());
|
||||
child.once("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function makeParentService(storePath: string, runCommandJob = vi.fn()) {
|
||||
return new CronService({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
runCommandJob,
|
||||
});
|
||||
}
|
||||
|
||||
describe("cron durable run ownership", () => {
|
||||
it("does not execute when the durable receipt cannot be recorded", async () => {
|
||||
vi.useRealTimers();
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.now();
|
||||
const job = makeCommandJob("receipt-required", now + 60_000);
|
||||
await saveCronStore(storePath, { version: 1, jobs: [job] });
|
||||
listCronRunReceipts(storePath, job.id);
|
||||
const database = openOpenClawStateDatabase().db;
|
||||
database.exec(`
|
||||
CREATE TRIGGER reject_cron_run_receipt
|
||||
BEFORE INSERT ON cron_run_receipts
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'receipt unavailable');
|
||||
END;
|
||||
`);
|
||||
const runner = vi.fn(async () => ({ status: "ok" as const }));
|
||||
const cron = makeParentService(storePath, runner);
|
||||
try {
|
||||
await expect(cron.run(job.id, "force")).rejects.toThrow("receipt unavailable");
|
||||
expect(runner).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
cron.stop();
|
||||
database.exec("DROP TRIGGER IF EXISTS reject_cron_run_receipt");
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps a live run fenced across an overlapping gateway start", async () => {
|
||||
vi.useRealTimers();
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.now();
|
||||
const job = makeCommandJob("restart-mid-run", now + 60_000);
|
||||
await saveCronStore(storePath, { version: 1, jobs: [job] });
|
||||
const releasePath = path.join(scriptRoot, `release-${now}`);
|
||||
const outputPath = path.join(scriptRoot, `output-${now}`);
|
||||
const owner = spawnRunner({ storePath, jobId: job.id, mode: "block", releasePath, outputPath });
|
||||
await waitForLine(owner, "started");
|
||||
|
||||
const replacementRunner = vi.fn(async () => ({ status: "ok" as const }));
|
||||
const replacement = makeParentService(storePath, replacementRunner);
|
||||
await replacement.start();
|
||||
await expect(replacement.run(job.id, "force")).resolves.toEqual({
|
||||
ok: true,
|
||||
ran: false,
|
||||
reason: "already-running",
|
||||
});
|
||||
expect(replacementRunner).not.toHaveBeenCalled();
|
||||
expect(listCronRunReceipts(storePath, job.id)).toMatchObject([{ status: "running" }]);
|
||||
replacement.stop();
|
||||
|
||||
owner.kill("SIGKILL");
|
||||
await waitForExit(owner);
|
||||
const recovered = makeParentService(storePath);
|
||||
await recovered.start();
|
||||
recovered.stop();
|
||||
|
||||
expect(listCronRunReceipts(storePath, job.id)[0]).toMatchObject({ status: "interrupted" });
|
||||
expect((await loadCronStore(storePath)).jobs[0]?.state.lastError).toContain(
|
||||
"interrupted by gateway restart",
|
||||
);
|
||||
});
|
||||
|
||||
it("admits one payload across overlapping scheduler processes", async () => {
|
||||
vi.useRealTimers();
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.now();
|
||||
const job = makeCommandJob("overlapping-ticks", now - 1);
|
||||
await saveCronStore(storePath, { version: 1, jobs: [job] });
|
||||
const releasePath = path.join(scriptRoot, `barrier-${now}`);
|
||||
const outputPath = path.join(scriptRoot, `ticks-${now}`);
|
||||
const first = spawnRunner({ storePath, jobId: job.id, mode: "due", releasePath, outputPath });
|
||||
const second = spawnRunner({ storePath, jobId: job.id, mode: "due", releasePath, outputPath });
|
||||
await Promise.all([waitForExit(first), waitForExit(second)]);
|
||||
|
||||
const invocations = fs.existsSync(outputPath)
|
||||
? fs.readFileSync(outputPath, "utf8").trim().split("\n").filter(Boolean)
|
||||
: [];
|
||||
expect(invocations).toHaveLength(1);
|
||||
expect(listCronRunReceipts(storePath, job.id)).toMatchObject([{ status: "ok" }]);
|
||||
});
|
||||
|
||||
it("supersedes a live run before payload effects after its owner changes", async () => {
|
||||
vi.useRealTimers();
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.now();
|
||||
const job = makeCommandJob("owner-change-live", now - 1, true);
|
||||
await saveCronStore(storePath, { version: 1, jobs: [job] });
|
||||
const releasePath = path.join(scriptRoot, `owner-release-${now}`);
|
||||
const outputPath = path.join(scriptRoot, `owner-output-${now}`);
|
||||
const owner = spawnRunner({
|
||||
storePath,
|
||||
jobId: job.id,
|
||||
mode: "trigger",
|
||||
releasePath,
|
||||
outputPath,
|
||||
});
|
||||
await waitForLine(owner, "trigger");
|
||||
|
||||
const editor = makeParentService(storePath);
|
||||
await editor.update(job.id, { agentId: "beta" });
|
||||
editor.stop();
|
||||
await fsPromises.writeFile(releasePath, "release");
|
||||
await waitForExit(owner);
|
||||
|
||||
expect(fs.existsSync(outputPath)).toBe(false);
|
||||
expect(listCronRunReceipts(storePath, job.id)[0]).toMatchObject({
|
||||
agentId: "alpha",
|
||||
status: "superseded",
|
||||
});
|
||||
const current = (await loadCronStore(storePath)).jobs[0];
|
||||
expect(current?.agentId).toBe("beta");
|
||||
expect(current?.state.lastRunAtMs).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,20 @@
|
||||
import { DEFAULT_CRON_MAX_CONCURRENT_RUNS } from "../../config/cron-limits.js";
|
||||
import { markCronJobActive } from "../active-jobs.js";
|
||||
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
|
||||
import {
|
||||
CronRunReceiptConflictError,
|
||||
CronRunReceiptRevisionError,
|
||||
finishCronRunReceipt,
|
||||
} from "../store/run-receipt-store.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
import {
|
||||
claimServiceCronRunReceipt,
|
||||
cronRunReceiptPersistHooks,
|
||||
supersedeServiceCronRunReceipt,
|
||||
} from "./run-receipts.js";
|
||||
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
|
||||
import { ensureLoaded, persistOrRestore, snapshotStoreForRollback } from "./store.js";
|
||||
import { tryCreateCronTaskRun } from "./task-runs.js";
|
||||
@@ -243,11 +253,28 @@ export async function activateQueuedCronRun(params: {
|
||||
onUnavailable?: () => void;
|
||||
onUnavailableRollbackError?: () => Promise<void>;
|
||||
}): Promise<
|
||||
| { kind: "activated"; startedAt: number }
|
||||
| {
|
||||
kind: "activated";
|
||||
startedAt: number;
|
||||
runReceipt: ReturnType<typeof claimServiceCronRunReceipt>;
|
||||
}
|
||||
| { kind: "fenced" }
|
||||
| { kind: "unavailable"; reason: "stopped" | "restart-recovery-pending" }
|
||||
> {
|
||||
const { state, job, reservationIdentity } = params;
|
||||
const startedAt = state.deps.nowMs();
|
||||
let runReceipt: ReturnType<typeof claimServiceCronRunReceipt>;
|
||||
try {
|
||||
runReceipt = claimServiceCronRunReceipt({ state, job, startedAtMs: startedAt });
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof CronRunReceiptConflictError ||
|
||||
error instanceof CronRunReceiptRevisionError
|
||||
) {
|
||||
return { kind: "fenced" };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const previousLastError = job.state.lastError;
|
||||
const activationRollbackSnapshot = snapshotStoreForRollback(state);
|
||||
delete job.state.queuedAtMs;
|
||||
@@ -255,14 +282,30 @@ export async function activateQueuedCronRun(params: {
|
||||
job.state.lastError = undefined;
|
||||
// Persist running ownership before execution. A failed write restores the
|
||||
// durable queued marker so the caller can release or recover that claim.
|
||||
await persistOrRestore(state, activationRollbackSnapshot);
|
||||
try {
|
||||
await persistOrRestore(state, activationRollbackSnapshot, {
|
||||
transactionHooks: cronRunReceiptPersistHooks({ state, handle: runReceipt }),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof CronRunReceiptRevisionError) {
|
||||
supersedeServiceCronRunReceipt(runReceipt, state.deps.nowMs(), error.message);
|
||||
return { kind: "fenced" };
|
||||
}
|
||||
finishCronRunReceipt({
|
||||
handle: runReceipt,
|
||||
status: "error",
|
||||
finishedAtMs: state.deps.nowMs(),
|
||||
error: normalizeCronRunErrorText(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
const reservation = state.queuedRunReservationsByJobId.get(job.id);
|
||||
if (reservation?.identity === reservationIdentity) {
|
||||
reservation.markerAtMs = startedAt;
|
||||
reservation.activationPreviousLastError = { value: previousLastError };
|
||||
}
|
||||
if (!state.stopped && !state.restartRecoveryPending) {
|
||||
return { kind: "activated", startedAt };
|
||||
return { kind: "activated", startedAt, runReceipt };
|
||||
}
|
||||
|
||||
params.onUnavailable?.();
|
||||
@@ -270,7 +313,17 @@ export async function activateQueuedCronRun(params: {
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
delete job.state.runningAtMs;
|
||||
try {
|
||||
await persistOrRestore(state, rollbackSnapshot);
|
||||
await persistOrRestore(state, rollbackSnapshot, {
|
||||
transactionHooks: cronRunReceiptPersistHooks({
|
||||
state,
|
||||
handle: runReceipt,
|
||||
terminal: {
|
||||
status: "skipped",
|
||||
finishedAtMs: state.deps.nowMs(),
|
||||
error: state.stopped ? "cron service stopped" : "cron restart recovery pending",
|
||||
},
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
await params.onUnavailableRollbackError?.();
|
||||
throw error;
|
||||
@@ -360,7 +413,7 @@ export async function executeQueuedCronRun(params: {
|
||||
}
|
||||
activated = true;
|
||||
params.onActivated?.();
|
||||
return { job, startedAt: activation.startedAt };
|
||||
return { job, startedAt: activation.startedAt, runReceipt: activation.runReceipt };
|
||||
});
|
||||
if (!started) {
|
||||
return undefined;
|
||||
@@ -372,6 +425,7 @@ export async function executeQueuedCronRun(params: {
|
||||
state,
|
||||
job: executionJob,
|
||||
startedAt: started.startedAt,
|
||||
publicRunId: started.runReceipt.receiptId,
|
||||
});
|
||||
const activeJobMarker = markCronJobActive(executionJob.id, {
|
||||
preserveAcrossGenerationAdvance: !runsDetachedFromMainSession(executionJob),
|
||||
@@ -389,12 +443,14 @@ export async function executeQueuedCronRun(params: {
|
||||
activeJobMarker,
|
||||
reservationIdentity: params.reservationIdentity,
|
||||
startedAt: started.startedAt,
|
||||
runReceipt: started.runReceipt,
|
||||
};
|
||||
let outcome: TimedCronRunOutcome;
|
||||
try {
|
||||
const result = await executeJobCoreWithTimeout(state, executionJob, {
|
||||
runId: taskRunId,
|
||||
activeJobMarker,
|
||||
runReceipt: started.runReceipt,
|
||||
});
|
||||
outcome = { ...base, ...result, endedAt: state.deps.nowMs() };
|
||||
} catch (error) {
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { CronStoreTransactionHooks } from "../store.js";
|
||||
import {
|
||||
assertCronRunReceiptCurrent,
|
||||
assertCronRunReceiptCurrentInDatabase,
|
||||
claimCronRunReceipt,
|
||||
CronRunReceiptRevisionError,
|
||||
finishCronRunReceipt,
|
||||
finishCronRunReceiptInDatabase,
|
||||
type CronRunReceiptHandle,
|
||||
type CronRunReceiptStatus,
|
||||
} from "../store/run-receipt-store.js";
|
||||
import type { CronJob, CronRunStatus } from "../types.js";
|
||||
import { resolveEffectiveJobAgentId } from "./ops-shared.js";
|
||||
import type { CronServiceState } from "./state.js";
|
||||
|
||||
function currentDefaultAgentId(state: CronServiceState): string | undefined {
|
||||
return state.deps.resolveDefaultAgentId?.() ?? state.deps.defaultAgentId;
|
||||
}
|
||||
|
||||
export function resolveCronRunReceiptAgentId(state: CronServiceState, job: CronJob): string {
|
||||
return resolveEffectiveJobAgentId(job, currentDefaultAgentId(state));
|
||||
}
|
||||
|
||||
function resolveAgentId(state: CronServiceState) {
|
||||
return (job: CronJob) => resolveCronRunReceiptAgentId(state, job);
|
||||
}
|
||||
|
||||
export function claimServiceCronRunReceipt(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
startedAtMs: number;
|
||||
requestRunId?: string;
|
||||
}): CronRunReceiptHandle {
|
||||
return claimCronRunReceipt({
|
||||
storePath: params.state.deps.storePath,
|
||||
job: params.job,
|
||||
agentId: resolveCronRunReceiptAgentId(params.state, params.job),
|
||||
startedAtMs: params.startedAtMs,
|
||||
requestRunId: params.requestRunId,
|
||||
resolveAgentId: resolveAgentId(params.state),
|
||||
});
|
||||
}
|
||||
|
||||
export function assertServiceCronRunReceiptCurrent(
|
||||
state: CronServiceState,
|
||||
handle: CronRunReceiptHandle,
|
||||
): void {
|
||||
assertCronRunReceiptCurrent({
|
||||
handle,
|
||||
resolveAgentId: resolveAgentId(state),
|
||||
isAgentAvailable: state.deps.isAgentAvailable,
|
||||
});
|
||||
}
|
||||
|
||||
function terminalReceiptStatus(status: CronRunStatus): Exclude<CronRunReceiptStatus, "running"> {
|
||||
if (status === "ok") {
|
||||
return "ok";
|
||||
}
|
||||
if (status === "skipped") {
|
||||
return "skipped";
|
||||
}
|
||||
return "error";
|
||||
}
|
||||
|
||||
export function cronRunReceiptPersistHooks(params: {
|
||||
state: CronServiceState;
|
||||
handle: CronRunReceiptHandle;
|
||||
terminal?: { status: CronRunStatus; finishedAtMs: number; error?: string };
|
||||
}): CronStoreTransactionHooks {
|
||||
return {
|
||||
beforeWrite: (database) => {
|
||||
if (params.state.deps.isAgentAvailable?.(params.handle.agentId) === false) {
|
||||
throw new CronRunReceiptRevisionError(
|
||||
params.handle.receiptId,
|
||||
`cron run owner ${params.handle.agentId} is no longer configured`,
|
||||
);
|
||||
}
|
||||
assertCronRunReceiptCurrentInDatabase({
|
||||
database,
|
||||
handle: params.handle,
|
||||
resolveAgentId: resolveAgentId(params.state),
|
||||
});
|
||||
},
|
||||
...(params.terminal
|
||||
? {
|
||||
afterWrite: (
|
||||
database: Parameters<NonNullable<CronStoreTransactionHooks["afterWrite"]>>[0],
|
||||
) => {
|
||||
finishCronRunReceiptInDatabase({
|
||||
database,
|
||||
handle: params.handle,
|
||||
status: terminalReceiptStatus(params.terminal!.status),
|
||||
finishedAtMs: params.terminal!.finishedAtMs,
|
||||
error: params.terminal!.error,
|
||||
});
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function supersedeServiceCronRunReceipt(
|
||||
handle: CronRunReceiptHandle,
|
||||
finishedAtMs: number,
|
||||
error: string,
|
||||
): void {
|
||||
finishCronRunReceipt({
|
||||
handle,
|
||||
status: "superseded",
|
||||
finishedAtMs,
|
||||
error,
|
||||
});
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
getCronJobsStoreRevision,
|
||||
loadCronJobsStoreWithConfigJobs,
|
||||
saveCronJobsStore,
|
||||
type CronStoreTransactionHooks,
|
||||
type QuarantinedCronConfigJob,
|
||||
} from "../store.js";
|
||||
import type { CronJob, CronStoreFile } from "../types.js";
|
||||
@@ -22,6 +23,7 @@ type PersistOptions = {
|
||||
stateOnly?: boolean;
|
||||
suppressScheduledJobId?: string;
|
||||
postPersistNotifications?: DeferredCronNotifications;
|
||||
transactionHooks?: CronStoreTransactionHooks;
|
||||
};
|
||||
|
||||
export type CronRollbackSnapshot = {
|
||||
@@ -282,7 +284,13 @@ export async function persist(state: CronServiceState, opts?: PersistOptions) {
|
||||
await saveCronJobsStore(
|
||||
state.deps.storePath,
|
||||
store,
|
||||
quarantine ? { quarantine } : stateOnly ? { stateOnly: true } : undefined,
|
||||
quarantine
|
||||
? { quarantine, transactionHooks: opts?.transactionHooks }
|
||||
: stateOnly
|
||||
? { stateOnly: true, transactionHooks: opts?.transactionHooks }
|
||||
: opts?.transactionHooks
|
||||
? { transactionHooks: opts.transactionHooks }
|
||||
: undefined,
|
||||
);
|
||||
} catch (error) {
|
||||
if (!quarantine) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { normalizeAgentId, resolveAgentIdFromSessionKey } from "../../routing/se
|
||||
import { deliveryContextFromSession } from "../../utils/delivery-context.shared.js";
|
||||
import type { DeliveryContext } from "../../utils/delivery-context.types.js";
|
||||
import type { CronActiveJobMarker } from "../active-jobs.js";
|
||||
import type { CronRunReceiptHandle } from "../store/run-receipt-store.js";
|
||||
import type {
|
||||
CronAgentExecutionPhaseUpdate,
|
||||
CronAgentExecutionStarted,
|
||||
@@ -46,6 +47,7 @@ export type TimedCronRunOutcome = CronRunOutcome &
|
||||
isolatedAgentSetupTimeout?: IsolatedAgentSetupTimeoutSignal;
|
||||
activeJobMarker?: CronActiveJobMarker;
|
||||
reservationIdentity?: object;
|
||||
runReceipt?: CronRunReceiptHandle;
|
||||
startedAt: number;
|
||||
endedAt: number;
|
||||
triggerEval?: CronTriggerEvalOutcome;
|
||||
@@ -110,6 +112,8 @@ export type ExecuteJobCoreOptions = {
|
||||
onExecutionStarted?: (info?: CronAgentExecutionStarted) => void;
|
||||
onExecutionPhase?: (info: CronAgentExecutionPhaseUpdate) => void;
|
||||
onLaneWait?: (info?: { waiting?: boolean }) => void;
|
||||
/** Revalidates the durable run fence after awaited planning and before effects. */
|
||||
assertRunCurrent?: () => void;
|
||||
streamBatch?: string;
|
||||
// Source definition and logical identity are an inseparable admission claim.
|
||||
// The key catches edits; the identity catches disable→re-enable and A→B→A.
|
||||
|
||||
@@ -141,6 +141,7 @@ export async function executeJobCore(
|
||||
effectiveJob = { ...job, payload: appendCronPayloadText(job.payload, evaluation.message) };
|
||||
}
|
||||
}
|
||||
options?.assertRunCurrent?.();
|
||||
if (effectiveJob.payload.kind === "script") {
|
||||
const result = await executeScriptCronJob(
|
||||
state,
|
||||
@@ -148,6 +149,7 @@ export async function executeJobCore(
|
||||
abortSignal,
|
||||
options?.activeJobMarker,
|
||||
options?.streamBatch,
|
||||
options?.assertRunCurrent,
|
||||
);
|
||||
return triggerEval ? { ...result, triggerEval } : result;
|
||||
}
|
||||
@@ -493,6 +495,7 @@ async function executeScriptCronJob(
|
||||
abortSignal: AbortSignal | undefined,
|
||||
activeJobMarker?: CronActiveJobMarker,
|
||||
streamBatch?: string,
|
||||
assertRunCurrent?: () => void,
|
||||
) {
|
||||
if (state.deps.cronConfig?.triggers?.enabled !== true) {
|
||||
return {
|
||||
@@ -513,6 +516,7 @@ async function executeScriptCronJob(
|
||||
if (abortSignal?.aborted) {
|
||||
return { status: "error" as const, error: abortErrorMessage(abortSignal) };
|
||||
}
|
||||
assertRunCurrent?.();
|
||||
if (result.status !== "ok") {
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
isSetupTimeoutErrorText,
|
||||
timeoutErrorMessage,
|
||||
} from "./execution-errors.js";
|
||||
import { assertServiceCronRunReceiptCurrent } from "./run-receipts.js";
|
||||
import type { CronServiceState } from "./state.js";
|
||||
import { tryUpdateCronTaskRunSession, withCronTaskRunId } from "./task-runs.js";
|
||||
import { resolveCronJobTimeoutMs } from "./timeout-policy.js";
|
||||
@@ -43,6 +44,7 @@ type CronCoreRunOptions = {
|
||||
streamBatch?: string;
|
||||
streamScheduleKey?: string;
|
||||
streamSourceIdentity?: string;
|
||||
runReceipt?: import("../store/run-receipt-store.js").CronRunReceiptHandle;
|
||||
};
|
||||
|
||||
async function deliverPrimaryWebhook(
|
||||
@@ -52,6 +54,7 @@ async function deliverPrimaryWebhook(
|
||||
abortSignal: AbortSignal,
|
||||
progress: CronRunProgress,
|
||||
deadlineAtMs?: number,
|
||||
assertRunCurrent?: () => void,
|
||||
): Promise<CronCoreRunOutcome> {
|
||||
const settle = (settledResult: CronCoreRunOutcome) => {
|
||||
// Publish the terminal delivery fact before this async function resolves;
|
||||
@@ -97,6 +100,8 @@ async function deliverPrimaryWebhook(
|
||||
});
|
||||
}
|
||||
|
||||
assertRunCurrent?.();
|
||||
|
||||
const startedAt = job.state.runningAtMs;
|
||||
const deliveredResult = withPrimaryWebhookTrace({ job, result, delivered: true });
|
||||
try {
|
||||
@@ -183,6 +188,9 @@ export async function executeJobCoreWithTimeout(
|
||||
opts?: CronCoreRunOptions,
|
||||
): Promise<CronCoreRunOutcome> {
|
||||
const runAbortController = new AbortController();
|
||||
const assertRunCurrent = opts?.runReceipt
|
||||
? () => assertServiceCronRunReceiptCurrent(state, opts.runReceipt!)
|
||||
: undefined;
|
||||
const operatorCancellationMarker = Symbol("cron-operator-cancelled");
|
||||
let resolveOperatorCancellation: ((value: typeof operatorCancellationMarker) => void) | undefined;
|
||||
const operatorCancellationPromise = new Promise<typeof operatorCancellationMarker>((resolve) => {
|
||||
@@ -244,13 +252,22 @@ export async function executeJobCoreWithTimeout(
|
||||
streamSourceIdentity: opts?.streamSourceIdentity,
|
||||
onExecutionStarted: noteExecutionStarted,
|
||||
onExecutionPhase: accumulateExecution,
|
||||
assertRunCurrent,
|
||||
};
|
||||
const corePromise = withCronTaskRunId(opts?.runId, () =>
|
||||
executeJobCore(state, job, runAbortController.signal, coreOptions),
|
||||
);
|
||||
const runPromise = corePromise.then(async (result) => {
|
||||
progress.completedCoreResult = result;
|
||||
return await deliverPrimaryWebhook(state, job, result, runAbortController.signal, progress);
|
||||
return await deliverPrimaryWebhook(
|
||||
state,
|
||||
job,
|
||||
result,
|
||||
runAbortController.signal,
|
||||
progress,
|
||||
undefined,
|
||||
assertRunCurrent,
|
||||
);
|
||||
});
|
||||
trackActiveCronTaskRunSettlement(runPromise, runAbortController.signal);
|
||||
void runPromise.catch((err: unknown) => {
|
||||
@@ -322,6 +339,7 @@ export async function executeJobCoreWithTimeout(
|
||||
onExecutionStarted: deferTimeoutUntilExecutionStart ? noteRunnerStarted : undefined,
|
||||
onExecutionPhase: deferTimeoutUntilExecutionStart ? watchdog.notePhase : undefined,
|
||||
onLaneWait: deferTimeoutUntilExecutionStart ? noteLaneState : undefined,
|
||||
assertRunCurrent,
|
||||
};
|
||||
const corePromise = withCronTaskRunId(opts?.runId, () =>
|
||||
executeJobCore(state, job, runAbortController.signal, coreOptions),
|
||||
@@ -336,6 +354,7 @@ export async function executeJobCoreWithTimeout(
|
||||
runAbortController.signal,
|
||||
progress,
|
||||
watchdog.deadlineAtMs(),
|
||||
assertRunCurrent,
|
||||
);
|
||||
});
|
||||
trackActiveCronTaskRunSettlement(runPromise, runAbortController.signal);
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
/** Finalizes cron task rows and active markers after timer outcome persistence. */
|
||||
import { clearCronJobActive, isCronActiveJobMarkerCurrent } from "../active-jobs.js";
|
||||
import type { CronActiveJobMarker } from "../active-jobs.js";
|
||||
import {
|
||||
CronRunReceiptRevisionError,
|
||||
releaseLocalCronRunReceiptOwnership,
|
||||
type CronRunReceiptHandle,
|
||||
} from "../store/run-receipt-store.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
import { releaseQueuedCronRun } from "./run-admission.js";
|
||||
import { cronRunReceiptPersistHooks, supersedeServiceCronRunReceipt } from "./run-receipts.js";
|
||||
import { emit, type CronServiceState, type DeferredCronNotifications } from "./state.js";
|
||||
import {
|
||||
ensureLoaded,
|
||||
@@ -26,6 +32,7 @@ type CronTaskRunFinalizationOutcome = {
|
||||
childSessionKey?: string;
|
||||
triggerEval?: { fired: boolean };
|
||||
activeJobMarker?: CronActiveJobMarker;
|
||||
runReceipt?: CronRunReceiptHandle;
|
||||
};
|
||||
|
||||
type CompletedCronRunOutcomeFinalizationOptions = {
|
||||
@@ -158,8 +165,37 @@ export async function finalizeCompletedCronRunOutcomes(
|
||||
);
|
||||
// Run notifications describe durable state. Drain them only after the
|
||||
// terminal write succeeds so rollback cannot publish a false outcome.
|
||||
const receiptHooks = finalizedOutcomes
|
||||
.filter((outcome) => outcome.runReceipt)
|
||||
.map((outcome) =>
|
||||
cronRunReceiptPersistHooks({
|
||||
state,
|
||||
handle: outcome.runReceipt!,
|
||||
terminal: {
|
||||
status: outcome.status,
|
||||
finishedAtMs: outcome.endedAt,
|
||||
error: outcome.error,
|
||||
},
|
||||
}),
|
||||
);
|
||||
await persistOrRestore(state, rollbackSnapshot, {
|
||||
postPersistNotifications,
|
||||
...(receiptHooks.length > 0
|
||||
? {
|
||||
transactionHooks: {
|
||||
beforeWrite: (database) => {
|
||||
for (const hooks of receiptHooks) {
|
||||
hooks.beforeWrite?.(database);
|
||||
}
|
||||
},
|
||||
afterWrite: (database) => {
|
||||
for (const hooks of receiptHooks) {
|
||||
hooks.afterWrite?.(database);
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
pruneCronJobScratchAfterCommit(
|
||||
state,
|
||||
@@ -173,6 +209,22 @@ export async function finalizeCompletedCronRunOutcomes(
|
||||
|
||||
finalizationSucceeded ||= finalizedOutcomes.length > 0;
|
||||
return finalizedOutcomes;
|
||||
} catch (error) {
|
||||
if (error instanceof CronRunReceiptRevisionError) {
|
||||
const stale = outcomes.find((outcome) => outcome.runReceipt?.receiptId === error.receiptId);
|
||||
if (stale?.runReceipt) {
|
||||
supersedeServiceCronRunReceipt(stale.runReceipt, state.deps.nowMs(), error.message);
|
||||
tryFinishCronTaskRunWithoutHistory(state, {
|
||||
taskRunId: stale.taskRunId,
|
||||
status: "skipped",
|
||||
error: error.message,
|
||||
endedAt: state.deps.nowMs(),
|
||||
});
|
||||
const remaining = outcomes.filter((outcome) => outcome !== stale);
|
||||
return await finalizeCompletedCronRunOutcomes(state, remaining, opts);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
for (const outcome of outcomes) {
|
||||
if (outcome.reservationIdentity) {
|
||||
@@ -182,6 +234,11 @@ export async function finalizeCompletedCronRunOutcomes(
|
||||
if (opts?.clearOnFailure !== false || finalizationSucceeded) {
|
||||
clearActiveMarkersForOutcomes(outcomes);
|
||||
}
|
||||
for (const outcome of outcomes) {
|
||||
if (outcome.runReceipt) {
|
||||
releaseLocalCronRunReceiptOwnership(outcome.runReceipt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,6 +273,13 @@ function finishRetiredCronTaskRuns<T extends CronTaskRunFinalizationOutcome>(
|
||||
const current = new Set(currentOutcomes);
|
||||
for (const outcome of outcomes) {
|
||||
if (!current.has(outcome)) {
|
||||
if (outcome.runReceipt) {
|
||||
supersedeServiceCronRunReceipt(
|
||||
outcome.runReceipt,
|
||||
state.deps.nowMs(),
|
||||
"cron run retired before its result became durable",
|
||||
);
|
||||
}
|
||||
tryFinishCronTaskRunWithoutHistory(state, outcome);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -236,11 +236,19 @@ type SaveCronStoreOptions = {
|
||||
stateOnly?: boolean;
|
||||
};
|
||||
|
||||
export type CronStoreTransactionHooks = {
|
||||
/** Runs inside the authoritative write transaction before cron rows change. */
|
||||
beforeWrite?: (db: DatabaseSync) => void;
|
||||
/** Runs after row changes but before the same transaction commits. */
|
||||
afterWrite?: (db: DatabaseSync) => void;
|
||||
};
|
||||
|
||||
type SaveCronJobsStoreOptions = SaveCronStoreOptions & {
|
||||
quarantine?: {
|
||||
entries: readonly (QuarantinedCronConfigJob | CronQuarantinedJob)[];
|
||||
nowMs: number;
|
||||
};
|
||||
transactionHooks?: CronStoreTransactionHooks;
|
||||
};
|
||||
|
||||
/** Persists cron jobs, or only mutable runtime state when stateOnly is set. */
|
||||
@@ -256,6 +264,7 @@ export async function saveCronJobsStore(
|
||||
assertCronStoreCanPersist(store);
|
||||
}
|
||||
runOpenClawStateWriteTransaction((database) => {
|
||||
opts?.transactionHooks?.beforeWrite?.(database.db);
|
||||
if (opts?.quarantine?.entries.length) {
|
||||
saveCronQuarantinedJobs({
|
||||
storePath: resolvedStorePath,
|
||||
@@ -268,10 +277,12 @@ export async function saveCronJobsStore(
|
||||
// quarantine and full replacement commit together or roll back together.
|
||||
if (stateOnly) {
|
||||
updateCronRuntimeRows(database.db, storeKey, store);
|
||||
opts?.transactionHooks?.afterWrite?.(database.db);
|
||||
return;
|
||||
}
|
||||
const normalizedJobs = replaceCronRows(database.db, storeKey, store);
|
||||
replaceCronRuntimeAuthorityRows({ db: database.db, storeKey, jobs: normalizedJobs });
|
||||
opts?.transactionHooks?.afterWrite?.(database.db);
|
||||
});
|
||||
noteCronJobsStoreCommit(storeKey);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { openOpenClawStateDatabase } from "../../state/openclaw-state-db.js";
|
||||
import { setupCronServiceSuite } from "../service.test-harness.js";
|
||||
import { saveCronStore } from "../store.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import {
|
||||
assertCronRunReceiptCurrent,
|
||||
claimCronRunReceipt,
|
||||
CronRunReceiptConflictError,
|
||||
CronRunReceiptRevisionError,
|
||||
finishCronRunReceipt,
|
||||
listCronRunReceipts,
|
||||
} from "./run-receipt-store.js";
|
||||
|
||||
const { makeStorePath } = setupCronServiceSuite({ prefix: "cron-run-receipt-" });
|
||||
|
||||
function makeJob(id: string, agentId = "alpha"): CronJob {
|
||||
return {
|
||||
id,
|
||||
agentId,
|
||||
name: id,
|
||||
enabled: true,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 1,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: id },
|
||||
state: {},
|
||||
};
|
||||
}
|
||||
|
||||
function claim(storePath: string, job: CronJob, startedAtMs: number) {
|
||||
return claimCronRunReceipt({
|
||||
storePath,
|
||||
job,
|
||||
agentId: job.agentId!,
|
||||
startedAtMs,
|
||||
resolveAgentId: (current) => current.agentId!,
|
||||
});
|
||||
}
|
||||
|
||||
describe("cron run receipt store", () => {
|
||||
it("records one durable active run and rejects an overlapping claimant", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const job = makeJob("overlap");
|
||||
await saveCronStore(storePath, { version: 1, jobs: [job] });
|
||||
|
||||
const first = claim(storePath, job, 100);
|
||||
|
||||
expect(() => claim(storePath, job, 101)).toThrow(CronRunReceiptConflictError);
|
||||
expect(listCronRunReceipts(storePath, job.id)).toMatchObject([
|
||||
{ receiptId: first.receiptId, status: "running", startedAtMs: 100 },
|
||||
]);
|
||||
|
||||
finishCronRunReceipt({ handle: first, status: "ok", finishedAtMs: 110 });
|
||||
const second = claim(storePath, job, 120);
|
||||
finishCronRunReceipt({ handle: second, status: "skipped", finishedAtMs: 121 });
|
||||
|
||||
expect(listCronRunReceipts(storePath, job.id).map((receipt) => receipt.status)).toEqual([
|
||||
"skipped",
|
||||
"ok",
|
||||
]);
|
||||
});
|
||||
|
||||
it("retires a provably dead process claim before admitting its successor", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const job = makeJob("restart");
|
||||
await saveCronStore(storePath, { version: 1, jobs: [job] });
|
||||
const abandoned = claim(storePath, job, 200);
|
||||
openOpenClawStateDatabase()
|
||||
.db.prepare("UPDATE cron_run_receipts SET owner_pid = ? WHERE receipt_id = ?")
|
||||
.run(2_147_483_647, abandoned.receiptId);
|
||||
|
||||
const replacement = claim(storePath, job, 220);
|
||||
|
||||
expect(replacement.receiptId).not.toBe(abandoned.receiptId);
|
||||
expect(listCronRunReceipts(storePath, job.id)).toMatchObject([
|
||||
{ receiptId: replacement.receiptId, status: "running" },
|
||||
{ receiptId: abandoned.receiptId, status: "interrupted" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects a live run after its durable owner revision changes", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const admitted = makeJob("owner-change", "alpha");
|
||||
await saveCronStore(storePath, { version: 1, jobs: [admitted] });
|
||||
const receipt = claim(storePath, admitted, 300);
|
||||
const reassigned = { ...admitted, agentId: "beta", updatedAtMs: 2 };
|
||||
await saveCronStore(storePath, { version: 1, jobs: [reassigned] });
|
||||
|
||||
expect(() =>
|
||||
assertCronRunReceiptCurrent({
|
||||
handle: receipt,
|
||||
resolveAgentId: (job) => job.agentId!,
|
||||
}),
|
||||
).toThrow(CronRunReceiptRevisionError);
|
||||
|
||||
finishCronRunReceipt({
|
||||
handle: receipt,
|
||||
status: "superseded",
|
||||
finishedAtMs: 310,
|
||||
error: "owner changed",
|
||||
});
|
||||
expect(listCronRunReceipts(storePath, admitted.id)[0]).toMatchObject({
|
||||
status: "superseded",
|
||||
agentId: "alpha",
|
||||
error: "owner changed",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,463 @@
|
||||
import crypto from "node:crypto";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { Selectable } from "kysely";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "../../infra/kysely-sync.js";
|
||||
import { getFileLockProcessStartTime, isPidDefinitelyDead } from "../../shared/pid-alive.js";
|
||||
import type { DB as OpenClawStateDatabase } from "../../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
runOpenClawStateWriteTransaction,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "../../state/openclaw-state-db.js";
|
||||
import { OPENCLAW_STATE_SCHEMA_SQL } from "../../state/openclaw-state-schema.js";
|
||||
import { resolveCronJobConfigRevision } from "../config-revision.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { cronStoreKey } from "./key.js";
|
||||
import { loadedCronStoreFromRows, loadCronRows } from "./row-codec.js";
|
||||
|
||||
type CronRunReceiptDatabase = Pick<OpenClawStateDatabase, "cron_run_receipts">;
|
||||
type CronRunReceiptRow = Selectable<CronRunReceiptDatabase["cron_run_receipts"]>;
|
||||
|
||||
export type CronRunReceiptStatus =
|
||||
| "running"
|
||||
| "ok"
|
||||
| "error"
|
||||
| "skipped"
|
||||
| "interrupted"
|
||||
| "superseded";
|
||||
|
||||
export type CronRunReceipt = {
|
||||
receiptId: string;
|
||||
storeKey: string;
|
||||
jobId: string;
|
||||
configRevision: string;
|
||||
agentId: string;
|
||||
requestRunId?: string;
|
||||
status: CronRunReceiptStatus;
|
||||
ownerPid: number;
|
||||
ownerStartTime: number | null;
|
||||
startedAtMs: number;
|
||||
finishedAtMs: number | null;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type CronRunReceiptHandle = Pick<
|
||||
CronRunReceipt,
|
||||
| "agentId"
|
||||
| "configRevision"
|
||||
| "jobId"
|
||||
| "ownerPid"
|
||||
| "ownerStartTime"
|
||||
| "receiptId"
|
||||
| "startedAtMs"
|
||||
| "storeKey"
|
||||
>;
|
||||
|
||||
type ResolveReceiptAgentId = (job: CronJob) => string;
|
||||
|
||||
const CRON_RUN_RECEIPT_SCHEMA_START = "CREATE TABLE IF NOT EXISTS cron_run_receipts (";
|
||||
const CRON_RUN_RECEIPT_SCHEMA_END =
|
||||
"ON cron_run_receipts(store_key, job_id, started_at_ms DESC, receipt_id DESC);";
|
||||
const initializedDatabases = new WeakSet<DatabaseSync>();
|
||||
const locallyOwnedReceipts = new Set<string>();
|
||||
|
||||
export class CronRunReceiptConflictError extends Error {
|
||||
constructor(readonly receipt: CronRunReceipt) {
|
||||
super(`cron job ${receipt.jobId} is already running in process ${receipt.ownerPid}`);
|
||||
this.name = "CronRunReceiptConflictError";
|
||||
}
|
||||
}
|
||||
|
||||
export class CronRunReceiptRevisionError extends Error {
|
||||
constructor(
|
||||
readonly receiptId: string,
|
||||
message = "cron run configuration changed",
|
||||
) {
|
||||
super(message);
|
||||
this.name = "CronRunReceiptRevisionError";
|
||||
}
|
||||
}
|
||||
|
||||
function ensureCronRunReceiptSchema(database: DatabaseSync): void {
|
||||
const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf(CRON_RUN_RECEIPT_SCHEMA_START);
|
||||
const endMarker = OPENCLAW_STATE_SCHEMA_SQL.indexOf(CRON_RUN_RECEIPT_SCHEMA_END, start);
|
||||
if (start < 0 || endMarker < start) {
|
||||
throw new Error("OpenClaw cron run receipt schema marker is missing.");
|
||||
}
|
||||
database.exec(
|
||||
OPENCLAW_STATE_SCHEMA_SQL.slice(start, endMarker + CRON_RUN_RECEIPT_SCHEMA_END.length),
|
||||
); // sqlite-allow-raw -- Canonical feature-local additive DDL only.
|
||||
}
|
||||
|
||||
function query(database: DatabaseSync) {
|
||||
return getNodeSqliteKysely<CronRunReceiptDatabase>(database);
|
||||
}
|
||||
|
||||
function withReceiptWrite<T>(
|
||||
operationLabel: string,
|
||||
options: OpenClawStateDatabaseOptions,
|
||||
operation: (database: DatabaseSync) => T,
|
||||
): T {
|
||||
let initializedDatabase: DatabaseSync | undefined;
|
||||
const result = runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
if (!initializedDatabases.has(db)) {
|
||||
ensureCronRunReceiptSchema(db);
|
||||
initializedDatabase = db;
|
||||
}
|
||||
return operation(db);
|
||||
},
|
||||
options,
|
||||
{ operationLabel },
|
||||
);
|
||||
if (initializedDatabase) {
|
||||
initializedDatabases.add(initializedDatabase);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function isReceiptStatus(value: string): value is CronRunReceiptStatus {
|
||||
return (
|
||||
value === "running" ||
|
||||
value === "ok" ||
|
||||
value === "error" ||
|
||||
value === "skipped" ||
|
||||
value === "interrupted" ||
|
||||
value === "superseded"
|
||||
);
|
||||
}
|
||||
|
||||
function receiptFromRow(row: CronRunReceiptRow): CronRunReceipt {
|
||||
if (!isReceiptStatus(row.status)) {
|
||||
throw new Error(`invalid cron run receipt status ${row.status}`);
|
||||
}
|
||||
return {
|
||||
receiptId: row.receipt_id,
|
||||
storeKey: row.store_key,
|
||||
jobId: row.job_id,
|
||||
configRevision: row.config_revision,
|
||||
agentId: row.agent_id,
|
||||
...(row.request_run_id ? { requestRunId: row.request_run_id } : {}),
|
||||
status: row.status,
|
||||
ownerPid: row.owner_pid,
|
||||
ownerStartTime: row.owner_start_time,
|
||||
startedAtMs: row.started_at_ms,
|
||||
finishedAtMs: row.finished_at_ms,
|
||||
...(row.error_text ? { error: row.error_text } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function activeRow(database: DatabaseSync, storeKey: string, jobId: string) {
|
||||
return executeSqliteQueryTakeFirstSync(
|
||||
database,
|
||||
query(database)
|
||||
.selectFrom("cron_run_receipts")
|
||||
.selectAll()
|
||||
.where("store_key", "=", storeKey)
|
||||
.where("job_id", "=", jobId)
|
||||
.where("status", "=", "running"),
|
||||
);
|
||||
}
|
||||
|
||||
function currentJob(database: DatabaseSync, storeKey: string, jobId: string): CronJob | undefined {
|
||||
const rows = loadCronRows(database, storeKey);
|
||||
if (rows.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return loadedCronStoreFromRows(rows).store.jobs.find((job) => job.id === jobId);
|
||||
}
|
||||
|
||||
function sameOwner(left: CronRunReceiptRow, right: CronRunReceiptRow): boolean {
|
||||
return (
|
||||
left.receipt_id === right.receipt_id &&
|
||||
left.owner_pid === right.owner_pid &&
|
||||
left.owner_start_time === right.owner_start_time &&
|
||||
left.status === right.status
|
||||
);
|
||||
}
|
||||
|
||||
function ownerDefinitelyStale(row: CronRunReceiptRow): boolean {
|
||||
if (row.owner_pid === process.pid) {
|
||||
return !locallyOwnedReceipts.has(row.receipt_id);
|
||||
}
|
||||
if (isPidDefinitelyDead(row.owner_pid)) {
|
||||
return true;
|
||||
}
|
||||
const observedStartTime = getFileLockProcessStartTime(row.owner_pid);
|
||||
return (
|
||||
row.owner_start_time !== null &&
|
||||
observedStartTime !== null &&
|
||||
row.owner_start_time !== observedStartTime
|
||||
);
|
||||
}
|
||||
|
||||
function validateCurrentJob(params: {
|
||||
database: DatabaseSync;
|
||||
handle: Pick<
|
||||
CronRunReceiptHandle,
|
||||
"agentId" | "configRevision" | "jobId" | "receiptId" | "storeKey"
|
||||
>;
|
||||
resolveAgentId: ResolveReceiptAgentId;
|
||||
}): CronJob {
|
||||
const job = currentJob(params.database, params.handle.storeKey, params.handle.jobId);
|
||||
if (!job) {
|
||||
throw new CronRunReceiptRevisionError(params.handle.receiptId, "cron job was removed");
|
||||
}
|
||||
if (params.resolveAgentId(job) !== params.handle.agentId) {
|
||||
throw new CronRunReceiptRevisionError(params.handle.receiptId);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
function receiptHandle(receipt: CronRunReceipt): CronRunReceiptHandle {
|
||||
return {
|
||||
receiptId: receipt.receiptId,
|
||||
storeKey: receipt.storeKey,
|
||||
jobId: receipt.jobId,
|
||||
configRevision: receipt.configRevision,
|
||||
agentId: receipt.agentId,
|
||||
ownerPid: receipt.ownerPid,
|
||||
ownerStartTime: receipt.ownerStartTime,
|
||||
startedAtMs: receipt.startedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
/** Atomically records a run start and acquires the one-active-run durable fence. */
|
||||
export function claimCronRunReceipt(params: {
|
||||
storePath: string;
|
||||
job: CronJob;
|
||||
agentId: string;
|
||||
startedAtMs: number;
|
||||
requestRunId?: string;
|
||||
resolveAgentId: ResolveReceiptAgentId;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): CronRunReceiptHandle {
|
||||
const storeKey = cronStoreKey(params.storePath);
|
||||
const options = params.env ? { env: params.env } : {};
|
||||
const observed = withReceiptWrite("cron.run-receipt.inspect", options, (database) =>
|
||||
activeRow(database, storeKey, params.job.id),
|
||||
);
|
||||
const observedStale = observed ? ownerDefinitelyStale(observed) : false;
|
||||
const receiptId = crypto.randomUUID();
|
||||
const handle: CronRunReceiptHandle = {
|
||||
receiptId,
|
||||
storeKey,
|
||||
jobId: params.job.id,
|
||||
configRevision: resolveCronJobConfigRevision(params.job),
|
||||
agentId: params.agentId,
|
||||
ownerPid: process.pid,
|
||||
ownerStartTime: getFileLockProcessStartTime(process.pid),
|
||||
startedAtMs: params.startedAtMs,
|
||||
};
|
||||
|
||||
const claimed = withReceiptWrite("cron.run-receipt.claim", options, (database) => {
|
||||
const current = activeRow(database, storeKey, params.job.id);
|
||||
if (current) {
|
||||
if (observed && observedStale && sameOwner(current, observed)) {
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
query(database)
|
||||
.updateTable("cron_run_receipts")
|
||||
.set({
|
||||
status: "interrupted",
|
||||
finished_at_ms: params.startedAtMs,
|
||||
error_text: "cron: job interrupted by owner process exit",
|
||||
})
|
||||
.where("receipt_id", "=", current.receipt_id)
|
||||
.where("status", "=", "running"),
|
||||
);
|
||||
} else {
|
||||
throw new CronRunReceiptConflictError(receiptFromRow(current));
|
||||
}
|
||||
}
|
||||
validateCurrentJob({ database, handle, resolveAgentId: params.resolveAgentId });
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
query(database)
|
||||
.insertInto("cron_run_receipts")
|
||||
.values({
|
||||
receipt_id: receiptId,
|
||||
store_key: storeKey,
|
||||
job_id: params.job.id,
|
||||
config_revision: handle.configRevision,
|
||||
agent_id: params.agentId,
|
||||
request_run_id: params.requestRunId ?? null,
|
||||
status: "running",
|
||||
owner_pid: handle.ownerPid,
|
||||
owner_start_time: handle.ownerStartTime,
|
||||
started_at_ms: params.startedAtMs,
|
||||
finished_at_ms: null,
|
||||
error_text: null,
|
||||
}),
|
||||
);
|
||||
return receiptHandle(receiptFromRow(activeRow(database, storeKey, params.job.id)!));
|
||||
});
|
||||
locallyOwnedReceipts.add(claimed.receiptId);
|
||||
return claimed;
|
||||
}
|
||||
|
||||
/** Synchronous transaction guard used immediately before a run side effect or state write. */
|
||||
export function assertCronRunReceiptCurrentInDatabase(params: {
|
||||
database: DatabaseSync;
|
||||
handle: CronRunReceiptHandle;
|
||||
resolveAgentId: ResolveReceiptAgentId;
|
||||
}): void {
|
||||
const current = activeRow(params.database, params.handle.storeKey, params.handle.jobId);
|
||||
if (
|
||||
!current ||
|
||||
current.receipt_id !== params.handle.receiptId ||
|
||||
current.owner_pid !== params.handle.ownerPid ||
|
||||
current.owner_start_time !== params.handle.ownerStartTime
|
||||
) {
|
||||
throw new CronRunReceiptRevisionError(
|
||||
params.handle.receiptId,
|
||||
"cron run fence is no longer current",
|
||||
);
|
||||
}
|
||||
validateCurrentJob({
|
||||
database: params.database,
|
||||
handle: params.handle,
|
||||
resolveAgentId: params.resolveAgentId,
|
||||
});
|
||||
}
|
||||
|
||||
export function assertCronRunReceiptCurrent(params: {
|
||||
handle: CronRunReceiptHandle;
|
||||
resolveAgentId: ResolveReceiptAgentId;
|
||||
isAgentAvailable?: (agentId: string) => boolean;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): void {
|
||||
if (params.isAgentAvailable && !params.isAgentAvailable(params.handle.agentId)) {
|
||||
throw new CronRunReceiptRevisionError(
|
||||
params.handle.receiptId,
|
||||
`cron run owner ${params.handle.agentId} is no longer configured`,
|
||||
);
|
||||
}
|
||||
withReceiptWrite(
|
||||
"cron.run-receipt.assert-current",
|
||||
params.env ? { env: params.env } : {},
|
||||
(database) =>
|
||||
assertCronRunReceiptCurrentInDatabase({
|
||||
database,
|
||||
handle: params.handle,
|
||||
resolveAgentId: params.resolveAgentId,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function finishCronRunReceipt(params: {
|
||||
handle: CronRunReceiptHandle;
|
||||
status: Exclude<CronRunReceiptStatus, "running">;
|
||||
finishedAtMs: number;
|
||||
error?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): CronRunReceipt | undefined {
|
||||
const finished = withReceiptWrite(
|
||||
"cron.run-receipt.finish",
|
||||
params.env ? { env: params.env } : {},
|
||||
(database) => finishCronRunReceiptInDatabase({ database, ...params }),
|
||||
);
|
||||
locallyOwnedReceipts.delete(params.handle.receiptId);
|
||||
return finished;
|
||||
}
|
||||
|
||||
/** Releases only this process's liveness proof after terminal persistence fails. */
|
||||
export function releaseLocalCronRunReceiptOwnership(handle: CronRunReceiptHandle): void {
|
||||
locallyOwnedReceipts.delete(handle.receiptId);
|
||||
}
|
||||
|
||||
/** Completes the exact active receipt inside its caller's cron-state transaction. */
|
||||
export function finishCronRunReceiptInDatabase(params: {
|
||||
database: DatabaseSync;
|
||||
handle: CronRunReceiptHandle;
|
||||
status: Exclude<CronRunReceiptStatus, "running">;
|
||||
finishedAtMs: number;
|
||||
error?: string;
|
||||
}): CronRunReceipt | undefined {
|
||||
executeSqliteQuerySync(
|
||||
params.database,
|
||||
query(params.database)
|
||||
.updateTable("cron_run_receipts")
|
||||
.set({
|
||||
status: params.status,
|
||||
finished_at_ms: params.finishedAtMs,
|
||||
error_text: params.error ?? null,
|
||||
})
|
||||
.where("receipt_id", "=", params.handle.receiptId)
|
||||
.where("status", "=", "running")
|
||||
.where("owner_pid", "=", params.handle.ownerPid),
|
||||
);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
params.database,
|
||||
query(params.database)
|
||||
.selectFrom("cron_run_receipts")
|
||||
.selectAll()
|
||||
.where("receipt_id", "=", params.handle.receiptId),
|
||||
);
|
||||
return row ? receiptFromRow(row) : undefined;
|
||||
}
|
||||
|
||||
/** Returns a still-live foreign claim, or retires a provably stale owner. */
|
||||
export function reconcileCronRunReceiptForStartup(params: {
|
||||
storePath: string;
|
||||
jobId: string;
|
||||
startedAtMs: number;
|
||||
nowMs: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): CronRunReceipt | undefined {
|
||||
const storeKey = cronStoreKey(params.storePath);
|
||||
const options = params.env ? { env: params.env } : {};
|
||||
const observed = withReceiptWrite("cron.run-receipt.startup-inspect", options, (database) =>
|
||||
activeRow(database, storeKey, params.jobId),
|
||||
);
|
||||
if (!observed || observed.started_at_ms !== params.startedAtMs) {
|
||||
return undefined;
|
||||
}
|
||||
const stale = ownerDefinitelyStale(observed);
|
||||
return withReceiptWrite("cron.run-receipt.startup-reconcile", options, (database) => {
|
||||
const current = activeRow(database, storeKey, params.jobId);
|
||||
if (!current || !sameOwner(current, observed) || current.started_at_ms !== params.startedAtMs) {
|
||||
return current ? receiptFromRow(current) : undefined;
|
||||
}
|
||||
if (!stale) {
|
||||
return receiptFromRow(current);
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
query(database)
|
||||
.updateTable("cron_run_receipts")
|
||||
.set({
|
||||
status: "interrupted",
|
||||
finished_at_ms: params.nowMs,
|
||||
error_text: "cron: job interrupted by owner process exit",
|
||||
})
|
||||
.where("receipt_id", "=", current.receipt_id)
|
||||
.where("status", "=", "running"),
|
||||
);
|
||||
return undefined;
|
||||
});
|
||||
}
|
||||
|
||||
/** Stable history order for operator and recovery reads. */
|
||||
export function listCronRunReceipts(
|
||||
storePath: string,
|
||||
jobId?: string,
|
||||
env?: NodeJS.ProcessEnv,
|
||||
): CronRunReceipt[] {
|
||||
return withReceiptWrite("cron.run-receipt.list", env ? { env } : {}, (database) => {
|
||||
let builder = query(database)
|
||||
.selectFrom("cron_run_receipts")
|
||||
.selectAll()
|
||||
.where("store_key", "=", cronStoreKey(storePath));
|
||||
if (jobId) {
|
||||
builder = builder.where("job_id", "=", jobId);
|
||||
}
|
||||
return executeSqliteQuerySync(
|
||||
database,
|
||||
builder.orderBy("started_at_ms", "desc").orderBy("receipt_id", "desc"),
|
||||
).rows.map(receiptFromRow);
|
||||
});
|
||||
}
|
||||
@@ -25,6 +25,7 @@ export const FIRST_USE_STATE_INDEXES = [
|
||||
// lazy ensures run; fold them into the next natural schema-version bump.
|
||||
export const LAZY_ADDITIVE_STATE_TABLES = [
|
||||
...FIRST_USE_STATE_TABLES,
|
||||
"cron_run_receipts",
|
||||
"cron_store_epochs",
|
||||
"model_catalog_remote",
|
||||
"secret_store_entries",
|
||||
@@ -41,6 +42,8 @@ export const LAZY_ADDITIVE_STATE_TABLES = [
|
||||
] as const;
|
||||
export const LAZY_ADDITIVE_STATE_INDEXES = [
|
||||
...FIRST_USE_STATE_INDEXES,
|
||||
"idx_cron_run_receipts_active_job",
|
||||
"idx_cron_run_receipts_job_history",
|
||||
"secret_store_entries_live_idx",
|
||||
] as const;
|
||||
/** Maximum time one synchronous SQLite call may wait for a lock. */
|
||||
|
||||
+16
@@ -478,6 +478,21 @@ export interface CronJobs {
|
||||
wake_mode: string;
|
||||
}
|
||||
|
||||
export interface CronRunReceipts {
|
||||
agent_id: string;
|
||||
config_revision: string;
|
||||
error_text: string | null;
|
||||
finished_at_ms: number | null;
|
||||
job_id: string;
|
||||
owner_pid: number;
|
||||
owner_start_time: number | null;
|
||||
receipt_id: string;
|
||||
request_run_id: string | null;
|
||||
started_at_ms: number;
|
||||
status: string;
|
||||
store_key: string;
|
||||
}
|
||||
|
||||
export interface CronStoreEpochs {
|
||||
store_epoch: Generated<number>;
|
||||
store_key: string;
|
||||
@@ -1708,6 +1723,7 @@ export interface DB {
|
||||
cron_job_runtime_authorities: CronJobRuntimeAuthorities;
|
||||
cron_job_scratch: CronJobScratch;
|
||||
cron_jobs: CronJobs;
|
||||
cron_run_receipts: CronRunReceipts;
|
||||
cron_store_epochs: CronStoreEpochs;
|
||||
current_conversation_bindings: CurrentConversationBindings;
|
||||
delivery_queue_entries: DeliveryQueueEntries;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getOpenClawStateRuntimeSchema } from "./openclaw-state-schema-compatibility.js";
|
||||
|
||||
describe("OpenClaw state runtime schema projection", () => {
|
||||
it("omits lazy additive tables and their unique indexes before first use", () => {
|
||||
const schema = getOpenClawStateRuntimeSchema({ includeVersionLazyAdditiveTables: false });
|
||||
|
||||
expect(schema).not.toContain("CREATE TABLE IF NOT EXISTS cron_run_receipts");
|
||||
expect(schema).not.toContain("idx_cron_run_receipts_active_job");
|
||||
expect(schema).not.toContain("idx_cron_run_receipts_job_history");
|
||||
});
|
||||
});
|
||||
@@ -68,7 +68,9 @@ export function getOpenClawStateRuntimeSchema(options: {
|
||||
schema = `${schema.slice(0, start)}${schema.slice(end + endMarker.length)}`;
|
||||
}
|
||||
for (const indexName of omittedIndexes) {
|
||||
const start = schema.indexOf(`CREATE INDEX IF NOT EXISTS ${indexName}`);
|
||||
const plainStart = schema.indexOf(`CREATE INDEX IF NOT EXISTS ${indexName}`);
|
||||
const uniqueStart = schema.indexOf(`CREATE UNIQUE INDEX IF NOT EXISTS ${indexName}`);
|
||||
const start = plainStart >= 0 ? plainStart : uniqueStart;
|
||||
const end = start >= 0 ? schema.indexOf(";", start) : -1;
|
||||
if (start < 0 || end < 0) {
|
||||
throw new Error(`lazy additive state schema index is missing for ${indexName}`);
|
||||
|
||||
@@ -1496,6 +1496,36 @@ CREATE INDEX IF NOT EXISTS idx_cron_jobs_agent_session
|
||||
ON cron_jobs(agent_id, session_key, updated_at DESC, job_id)
|
||||
WHERE agent_id IS NOT NULL OR session_key IS NOT NULL;
|
||||
|
||||
-- One owner-native receipt is also the durable execution fence. Receipts
|
||||
-- survive job deletion so operators can distinguish a run from log inference.
|
||||
CREATE TABLE IF NOT EXISTS cron_run_receipts (
|
||||
receipt_id TEXT PRIMARY KEY,
|
||||
store_key TEXT NOT NULL,
|
||||
job_id TEXT NOT NULL,
|
||||
config_revision TEXT NOT NULL,
|
||||
agent_id TEXT NOT NULL,
|
||||
request_run_id TEXT,
|
||||
status TEXT NOT NULL,
|
||||
owner_pid INTEGER NOT NULL,
|
||||
owner_start_time INTEGER,
|
||||
started_at_ms INTEGER NOT NULL,
|
||||
finished_at_ms INTEGER,
|
||||
error_text TEXT,
|
||||
CHECK (status IN ('running', 'ok', 'error', 'skipped', 'interrupted', 'superseded')),
|
||||
CHECK (
|
||||
(status = 'running' AND finished_at_ms IS NULL)
|
||||
OR
|
||||
(status != 'running' AND finished_at_ms IS NOT NULL)
|
||||
)
|
||||
) STRICT;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_cron_run_receipts_active_job
|
||||
ON cron_run_receipts(store_key, job_id)
|
||||
WHERE status = 'running';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cron_run_receipts_job_history
|
||||
ON cron_run_receipts(store_key, job_id, started_at_ms DESC, receipt_id DESC);
|
||||
|
||||
-- Runtime-private authority is independent of job_json so downgraded writers
|
||||
-- can rewrite recognized job config without erasing or silently widening it.
|
||||
CREATE TABLE IF NOT EXISTS cron_job_runtime_authorities (
|
||||
|
||||
Reference in New Issue
Block a user