mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
Fix disabled heartbeat one-shot cron retries (#92225)
* fix: retry disabled cron wake one-shots * fix: satisfy cron retry CI checks
This commit is contained in:
@@ -74,6 +74,28 @@ describe("CronService restart catch-up", () => {
|
||||
};
|
||||
}
|
||||
|
||||
function createOverdueDisabledHeartbeatOneShotRetry(id: string, nextRunAtMs: number): CronJob {
|
||||
return {
|
||||
id,
|
||||
name: `disabled-heartbeat-retry-${id}`,
|
||||
enabled: true,
|
||||
createdAtMs: nextRunAtMs - 60_000,
|
||||
updatedAtMs: nextRunAtMs - 30_000,
|
||||
deleteAfterRun: true,
|
||||
schedule: { kind: "at", at: new Date(nextRunAtMs - 30_000).toISOString() },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "now",
|
||||
payload: { kind: "systemEvent", text: `retry-${id}` },
|
||||
state: {
|
||||
nextRunAtMs,
|
||||
lastRunAtMs: nextRunAtMs - 30_000,
|
||||
lastRunStatus: "skipped",
|
||||
lastError: "disabled",
|
||||
consecutiveSkipped: 1,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function expectQueuedSystemEvent(
|
||||
enqueueSystemEvent: ReturnType<typeof vi.fn>,
|
||||
expectedText: string,
|
||||
@@ -658,4 +680,43 @@ describe("CronService restart catch-up", () => {
|
||||
|
||||
await store.cleanup();
|
||||
});
|
||||
|
||||
it("stagger-limits overdue disabled-heartbeat one-shot retries after restart", async () => {
|
||||
const store = await makeStorePath();
|
||||
const startNow = Date.parse("2025-12-13T17:00:00.000Z");
|
||||
|
||||
await writeStoreJobs(store.storePath, [
|
||||
createOverdueDisabledHeartbeatOneShotRetry("disabled-retry-0", startNow - 60_000),
|
||||
createOverdueDisabledHeartbeatOneShotRetry("disabled-retry-1", startNow - 45_000),
|
||||
]);
|
||||
|
||||
const enqueueSystemEvent = vi.fn();
|
||||
const requestHeartbeat = vi.fn();
|
||||
const state = createCronServiceState({
|
||||
cronEnabled: true,
|
||||
storePath: store.storePath,
|
||||
log: noopLogger,
|
||||
nowMs: () => startNow,
|
||||
enqueueSystemEvent,
|
||||
requestHeartbeat,
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
maxMissedJobsPerRestart: 1,
|
||||
missedJobStaggerMs: 5_000,
|
||||
});
|
||||
|
||||
await runMissedJobs(state);
|
||||
|
||||
expectQueuedSystemEvent(enqueueSystemEvent, "retry-disabled-retry-0");
|
||||
expect(requestHeartbeat).toHaveBeenCalledTimes(1);
|
||||
|
||||
const listedJobs = state.store?.jobs ?? [];
|
||||
expect(listedJobs.find((job) => job.id === "disabled-retry-0")).toBeUndefined();
|
||||
const deferred = listedJobs.find((job) => job.id === "disabled-retry-1");
|
||||
expect(deferred?.enabled).toBe(true);
|
||||
expect(deferred?.state.lastRunStatus).toBe("skipped");
|
||||
expect(deferred?.state.lastError).toBe("disabled");
|
||||
expect(deferred?.state.nextRunAtMs).toBe(startNow + 5_000);
|
||||
|
||||
await store.cleanup();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,13 @@ import {
|
||||
HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT,
|
||||
type HeartbeatRunResult,
|
||||
} from "../infra/heartbeat-wake.js";
|
||||
import {
|
||||
consumeSelectedSystemEventEntries,
|
||||
drainSystemEventEntries,
|
||||
enqueueSystemEventEntry,
|
||||
peekSystemEventEntries,
|
||||
resetSystemEventsForTest,
|
||||
} from "../infra/system-events.js";
|
||||
import type { CronEvent, CronServiceDeps } from "./service.js";
|
||||
import { CronService } from "./service.js";
|
||||
import {
|
||||
@@ -60,6 +67,8 @@ type CronHarnessOptions = {
|
||||
runIsolatedAgentJob?: CronServiceDeps["runIsolatedAgentJob"];
|
||||
runHeartbeatOnce?: NonNullable<CronServiceDeps["runHeartbeatOnce"]>;
|
||||
nowMs?: () => number;
|
||||
cronConfig?: CronServiceDeps["cronConfig"];
|
||||
useRemovableSystemEventQueue?: boolean;
|
||||
wakeNowHeartbeatBusyMaxWaitMs?: number;
|
||||
wakeNowHeartbeatBusyRetryDelayMs?: number;
|
||||
withEvents?: boolean;
|
||||
@@ -67,7 +76,25 @@ type CronHarnessOptions = {
|
||||
|
||||
async function createCronHarness(options: CronHarnessOptions = {}) {
|
||||
const store = await makeStorePath();
|
||||
const enqueueSystemEvent = vi.fn();
|
||||
const enqueueSystemEvent = options.useRemovableSystemEventQueue
|
||||
? vi.fn((text: string, opts?: Parameters<CronServiceDeps["enqueueSystemEvent"]>[1]) => {
|
||||
if (!opts?.sessionKey) {
|
||||
throw new Error("test removable queue requires a sessionKey");
|
||||
}
|
||||
const event = enqueueSystemEventEntry(text, {
|
||||
sessionKey: opts.sessionKey,
|
||||
contextKey: opts.contextKey,
|
||||
deliveryContext: opts.deliveryContext,
|
||||
});
|
||||
return event
|
||||
? {
|
||||
accepted: true,
|
||||
remove: () =>
|
||||
consumeSelectedSystemEventEntries(opts.sessionKey as string, [event]).length > 0,
|
||||
}
|
||||
: { accepted: false };
|
||||
})
|
||||
: vi.fn();
|
||||
const requestHeartbeat = vi.fn();
|
||||
const events = options.withEvents === false ? undefined : createCronEventHarness();
|
||||
|
||||
@@ -76,6 +103,7 @@ async function createCronHarness(options: CronHarnessOptions = {}) {
|
||||
cronEnabled: true,
|
||||
log: noopLogger,
|
||||
...(options.nowMs ? { nowMs: options.nowMs } : {}),
|
||||
...(options.cronConfig ? { cronConfig: options.cronConfig } : {}),
|
||||
...(options.wakeNowHeartbeatBusyMaxWaitMs !== undefined
|
||||
? { wakeNowHeartbeatBusyMaxWaitMs: options.wakeNowHeartbeatBusyMaxWaitMs }
|
||||
: {}),
|
||||
@@ -236,10 +264,23 @@ function expectQueuedCronHeartbeat(
|
||||
expectCronRunSessionKey(request?.sessionKey, params.jobId);
|
||||
}
|
||||
|
||||
function getPostedSystemEventSessionKeys(enqueueSystemEvent: ReturnType<typeof vi.fn>) {
|
||||
return enqueueSystemEvent.mock.calls
|
||||
.map(([, options]) => (options as { sessionKey?: string } | undefined)?.sessionKey)
|
||||
.filter((sessionKey): sessionKey is string => Boolean(sessionKey));
|
||||
}
|
||||
|
||||
function expectNoQueuedEvents(sessionKeys: readonly string[]) {
|
||||
for (const sessionKey of sessionKeys) {
|
||||
expect(peekSystemEventEntries(sessionKey)).toHaveLength(0);
|
||||
}
|
||||
}
|
||||
|
||||
async function stopCronAndCleanup(cron: CronService, store: { cleanup: () => Promise<void> }) {
|
||||
await cron.status();
|
||||
cron.stop();
|
||||
await store.cleanup();
|
||||
resetSystemEventsForTest();
|
||||
}
|
||||
|
||||
function createStartedCronService(
|
||||
@@ -449,6 +490,101 @@ describe("CronService", () => {
|
||||
await stopCronAndCleanup(cron, store);
|
||||
});
|
||||
|
||||
it("retries disabled one-shot main wakes without leaving failed-attempt system events", async () => {
|
||||
resetSystemEventsForTest();
|
||||
const atMs = Date.parse("2025-12-13T00:00:02.000Z");
|
||||
let now = atMs;
|
||||
const consumedTexts: string[] = [];
|
||||
const runHeartbeatOnce = vi.fn(
|
||||
async (opts?: Parameters<NonNullable<CronServiceDeps["runHeartbeatOnce"]>>[0]) => {
|
||||
if (runHeartbeatOnce.mock.calls.length < 3) {
|
||||
return { status: "skipped" as const, reason: "disabled" };
|
||||
}
|
||||
const sessionKey = opts?.sessionKey;
|
||||
if (sessionKey) {
|
||||
consumedTexts.push(...drainSystemEventEntries(sessionKey).map((event) => event.text));
|
||||
}
|
||||
return { status: "ran" as const, durationMs: 1 };
|
||||
},
|
||||
);
|
||||
const { store, cron, enqueueSystemEvent, requestHeartbeat } = await createCronHarness({
|
||||
runHeartbeatOnce,
|
||||
nowMs: () => now,
|
||||
useRemovableSystemEventQueue: true,
|
||||
withEvents: false,
|
||||
});
|
||||
const job = await addMainOneShotHelloJob(cron, {
|
||||
atMs,
|
||||
name: "one-shot disabled heartbeat retries cleanly",
|
||||
});
|
||||
|
||||
await cron.run(job.id, "due");
|
||||
let jobs = await cron.list({ includeDisabled: true });
|
||||
let updated = jobs.find((j) => j.id === job.id);
|
||||
expect(updated?.enabled).toBe(true);
|
||||
expect(updated?.state.lastStatus).toBe("skipped");
|
||||
expect(updated?.state.lastError).toBe("disabled");
|
||||
expect(updated?.state.consecutiveSkipped).toBe(1);
|
||||
expect(updated?.state.nextRunAtMs).toBe(atMs + 30_000);
|
||||
expectNoQueuedEvents(getPostedSystemEventSessionKeys(enqueueSystemEvent));
|
||||
|
||||
now = updated?.state.nextRunAtMs ?? now;
|
||||
await cron.run(job.id, "due");
|
||||
jobs = await cron.list({ includeDisabled: true });
|
||||
updated = jobs.find((j) => j.id === job.id);
|
||||
expect(updated?.enabled).toBe(true);
|
||||
expect(updated?.state.consecutiveSkipped).toBe(2);
|
||||
expect(updated?.state.nextRunAtMs).toBe(atMs + 90_000);
|
||||
expectNoQueuedEvents(getPostedSystemEventSessionKeys(enqueueSystemEvent));
|
||||
|
||||
now = updated?.state.nextRunAtMs ?? now;
|
||||
await cron.run(job.id, "due");
|
||||
|
||||
jobs = await cron.list({ includeDisabled: true });
|
||||
expect(jobs.find((j) => j.id === job.id)).toBeUndefined();
|
||||
expect(runHeartbeatOnce).toHaveBeenCalledTimes(3);
|
||||
expect(requestHeartbeat).not.toHaveBeenCalled();
|
||||
expect(consumedTexts).toEqual(["hello"]);
|
||||
expectNoQueuedEvents(getPostedSystemEventSessionKeys(enqueueSystemEvent));
|
||||
|
||||
await stopCronAndCleanup(cron, store);
|
||||
});
|
||||
|
||||
it("disables exhausted disabled-heartbeat one-shots without leaving queued events", async () => {
|
||||
resetSystemEventsForTest();
|
||||
const atMs = Date.parse("2025-12-13T00:00:02.000Z");
|
||||
const runHeartbeatOnce = vi.fn(async () => ({
|
||||
status: "skipped" as const,
|
||||
reason: "disabled",
|
||||
}));
|
||||
const { store, cron, enqueueSystemEvent, requestHeartbeat } = await createCronHarness({
|
||||
runHeartbeatOnce,
|
||||
nowMs: () => atMs,
|
||||
cronConfig: { retry: { maxAttempts: 0, backoffMs: [30_000] } },
|
||||
useRemovableSystemEventQueue: true,
|
||||
withEvents: false,
|
||||
});
|
||||
const job = await addMainOneShotHelloJob(cron, {
|
||||
atMs,
|
||||
name: "one-shot disabled heartbeat exhausted",
|
||||
});
|
||||
|
||||
await cron.run(job.id, "due");
|
||||
|
||||
const jobs = await cron.list({ includeDisabled: true });
|
||||
const updated = jobs.find((j) => j.id === job.id);
|
||||
expect(updated?.enabled).toBe(false);
|
||||
expect(updated?.state.lastStatus).toBe("skipped");
|
||||
expect(updated?.state.lastError).toBe("disabled");
|
||||
expect(updated?.state.consecutiveSkipped).toBe(1);
|
||||
expect(updated?.state.nextRunAtMs).toBeUndefined();
|
||||
expect(runHeartbeatOnce).toHaveBeenCalledTimes(1);
|
||||
expect(requestHeartbeat).not.toHaveBeenCalled();
|
||||
expectNoQueuedEvents(getPostedSystemEventSessionKeys(enqueueSystemEvent));
|
||||
|
||||
await stopCronAndCleanup(cron, store);
|
||||
});
|
||||
|
||||
it("runs an isolated job without posting a fallback summary to main", async () => {
|
||||
const runIsolatedAgentJob = vi.fn(async () => ({ status: "ok" as const, summary: "done" }));
|
||||
const { store, cron, enqueueSystemEvent, requestHeartbeat, events } =
|
||||
|
||||
@@ -51,6 +51,14 @@ export type Logger = {
|
||||
error: (obj: unknown, msg?: string) => void;
|
||||
};
|
||||
|
||||
export type CronSystemEventEnqueueResult =
|
||||
| boolean
|
||||
| void
|
||||
| {
|
||||
accepted?: boolean;
|
||||
remove?: () => boolean | void;
|
||||
};
|
||||
|
||||
/** Dependency injection surface for the cron service runtime. */
|
||||
export type CronServiceDeps = {
|
||||
nowMs?: () => number;
|
||||
@@ -90,7 +98,7 @@ export type CronServiceDeps = {
|
||||
contextKey?: string;
|
||||
deliveryContext?: DeliveryContext;
|
||||
},
|
||||
) => void;
|
||||
) => CronSystemEventEnqueueResult;
|
||||
/**
|
||||
* Resolve the channel-correct origin delivery context for a session key (the
|
||||
* value the channel's send expects, e.g. Telegram message_thread_id), sourced
|
||||
|
||||
+157
-19
@@ -63,7 +63,7 @@ import {
|
||||
resolveJobPayloadTextForMain,
|
||||
} from "./jobs.js";
|
||||
import { locked } from "./locked.js";
|
||||
import type { CronEvent, CronServiceState } from "./state.js";
|
||||
import type { CronEvent, CronServiceState, CronSystemEventEnqueueResult } from "./state.js";
|
||||
import { ensureLoaded, persist } from "./store.js";
|
||||
import {
|
||||
resolveMainSessionCronRunSessionKey,
|
||||
@@ -78,6 +78,7 @@ export { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
export { wake } from "./wake.js";
|
||||
|
||||
const MAX_TIMER_DELAY_MS = 60_000;
|
||||
const HEARTBEAT_SKIP_DISABLED = "disabled";
|
||||
|
||||
/**
|
||||
* Minimum gap between consecutive fires of the same cron job. This is a
|
||||
@@ -275,6 +276,18 @@ type TransientCronRetryDecision = {
|
||||
reason: "transient retry" | "max retries exhausted" | "permanent error";
|
||||
};
|
||||
|
||||
type DisabledHeartbeatOneShotRetryDecision = {
|
||||
retryable: boolean;
|
||||
consecutiveSkipped: number;
|
||||
backoffMs?: number;
|
||||
reason: "disabled heartbeat retry" | "max retries exhausted";
|
||||
};
|
||||
|
||||
type QueuedSystemEventHandle = {
|
||||
accepted: boolean;
|
||||
remove?: () => boolean | void;
|
||||
};
|
||||
|
||||
function resolveCronNextRunWithLowerBound(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
@@ -347,6 +360,98 @@ function resolveTransientCronRetryDecision(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDisabledHeartbeatOneShotRetryDecision(params: {
|
||||
cronConfig?: CronConfig;
|
||||
consecutiveSkipped: number | undefined;
|
||||
}): DisabledHeartbeatOneShotRetryDecision {
|
||||
const retryConfig = resolveRetryConfig(params.cronConfig);
|
||||
const consecutiveSkipped = params.consecutiveSkipped ?? 0;
|
||||
if (consecutiveSkipped > retryConfig.maxAttempts) {
|
||||
return {
|
||||
retryable: false,
|
||||
consecutiveSkipped,
|
||||
reason: "max retries exhausted",
|
||||
};
|
||||
}
|
||||
return {
|
||||
retryable: true,
|
||||
consecutiveSkipped,
|
||||
backoffMs: errorBackoffMs(consecutiveSkipped, retryConfig.backoffMs),
|
||||
reason: "disabled heartbeat retry",
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeQueuedSystemEventHandle(
|
||||
result: CronSystemEventEnqueueResult,
|
||||
): QueuedSystemEventHandle {
|
||||
if (typeof result === "boolean") {
|
||||
return { accepted: result };
|
||||
}
|
||||
if (result && typeof result === "object") {
|
||||
return {
|
||||
accepted: result.accepted !== false,
|
||||
...(result.remove ? { remove: result.remove } : {}),
|
||||
};
|
||||
}
|
||||
return { accepted: true };
|
||||
}
|
||||
|
||||
function removeQueuedSystemEventHandle(
|
||||
state: CronServiceState,
|
||||
job: CronJob,
|
||||
queued: QueuedSystemEventHandle,
|
||||
) {
|
||||
if (!queued.accepted || !queued.remove) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
queued.remove();
|
||||
} catch (err) {
|
||||
state.deps.log.warn(
|
||||
{ jobId: job.id, jobName: job.name, err },
|
||||
"cron: failed to remove undelivered main-session system event",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function shouldRetryDisabledHeartbeatOneShot(
|
||||
job: CronJob,
|
||||
result: { status: CronRunStatus; error?: string },
|
||||
): boolean {
|
||||
return (
|
||||
job.schedule.kind === "at" &&
|
||||
job.sessionTarget === "main" &&
|
||||
job.wakeMode === "now" &&
|
||||
result.status === "skipped" &&
|
||||
result.error === HEARTBEAT_SKIP_DISABLED
|
||||
);
|
||||
}
|
||||
|
||||
function isScheduledTerminalOneShotRetry(
|
||||
job: CronJob,
|
||||
lastRunStatus: CronRunStatus,
|
||||
lastRun: unknown,
|
||||
nextRun: unknown,
|
||||
): boolean {
|
||||
if (
|
||||
!isJobEnabled(job) ||
|
||||
typeof nextRun !== "number" ||
|
||||
typeof lastRun !== "number" ||
|
||||
nextRun <= lastRun
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (lastRunStatus === "error") {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
lastRunStatus === "skipped" &&
|
||||
job.sessionTarget === "main" &&
|
||||
job.wakeMode === "now" &&
|
||||
job.state.lastError === HEARTBEAT_SKIP_DISABLED
|
||||
);
|
||||
}
|
||||
|
||||
function resolveDeliveryState(params: {
|
||||
job: CronJob;
|
||||
runStatus: CronRunStatus;
|
||||
@@ -532,10 +637,42 @@ export function applyJobResult(
|
||||
|
||||
const shouldDelete =
|
||||
job.schedule.kind === "at" && job.deleteAfterRun === true && result.status === "ok";
|
||||
const retryDisabledHeartbeatOneShot = shouldRetryDisabledHeartbeatOneShot(job, result);
|
||||
|
||||
if (!shouldDelete) {
|
||||
if (job.schedule.kind === "at") {
|
||||
if (result.status === "ok" || result.status === "skipped") {
|
||||
if (retryDisabledHeartbeatOneShot) {
|
||||
const retryDecision = resolveDisabledHeartbeatOneShotRetryDecision({
|
||||
cronConfig: state.deps.cronConfig,
|
||||
consecutiveSkipped: job.state.consecutiveSkipped,
|
||||
});
|
||||
if (retryDecision.retryable && retryDecision.backoffMs !== undefined) {
|
||||
job.enabled = true;
|
||||
job.state.nextRunAtMs = result.endedAt + retryDecision.backoffMs;
|
||||
state.deps.log.info(
|
||||
{
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
consecutiveSkipped: retryDecision.consecutiveSkipped,
|
||||
backoffMs: retryDecision.backoffMs,
|
||||
nextRunAtMs: job.state.nextRunAtMs,
|
||||
},
|
||||
"cron: scheduling one-shot retry after disabled heartbeat",
|
||||
);
|
||||
} else {
|
||||
job.enabled = false;
|
||||
job.state.nextRunAtMs = undefined;
|
||||
state.deps.log.warn(
|
||||
{
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
consecutiveSkipped: retryDecision.consecutiveSkipped,
|
||||
reason: retryDecision.reason,
|
||||
},
|
||||
"cron: disabling one-shot job after disabled heartbeat retries",
|
||||
);
|
||||
}
|
||||
} else if (result.status === "ok" || result.status === "skipped") {
|
||||
// One-shot done or skipped: disable to prevent tight-loop (#11452).
|
||||
job.enabled = false;
|
||||
job.state.nextRunAtMs = undefined;
|
||||
@@ -1024,19 +1161,12 @@ function isRunnableJob(params: {
|
||||
}
|
||||
const lastRunStatus = resolveJobLastRunStatus(job);
|
||||
if (params.skipAtIfAlreadyRan && job.schedule.kind === "at" && lastRunStatus) {
|
||||
// One-shot with terminal status: skip unless it's a transient-error retry.
|
||||
// Retries have nextRunAtMs > lastRunAtMs (scheduled after the failed run) (#24355).
|
||||
// ok/skipped or error-without-retry always skip (#13845).
|
||||
// One-shot with terminal status: skip unless it has an explicit retry
|
||||
// scheduled after the failed/skipped run (#24355, #91775).
|
||||
const lastRun = job.state.lastRunAtMs;
|
||||
const nextRun = job.state.nextRunAtMs;
|
||||
if (
|
||||
lastRunStatus === "error" &&
|
||||
isJobEnabled(job) &&
|
||||
typeof nextRun === "number" &&
|
||||
typeof lastRun === "number" &&
|
||||
nextRun > lastRun
|
||||
) {
|
||||
return nowMs >= nextRun;
|
||||
if (isScheduledTerminalOneShotRetry(job, lastRunStatus, lastRun, nextRun)) {
|
||||
return typeof nextRun === "number" && nowMs >= nextRun;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -1450,12 +1580,14 @@ async function executeMainSessionCronJob(
|
||||
const deliveryContext = resolveMainSessionCronDeliveryContext(state, job);
|
||||
// Main-session jobs enqueue text into a per-run child session so each cron
|
||||
// execution has its own transcript and task drill-down target.
|
||||
state.deps.enqueueSystemEvent(text, {
|
||||
agentId: job.agentId,
|
||||
sessionKey: cronRunSessionKey,
|
||||
contextKey: `cron:${job.id}`,
|
||||
...(deliveryContext ? { deliveryContext } : {}),
|
||||
});
|
||||
const queuedSystemEvent = normalizeQueuedSystemEventHandle(
|
||||
state.deps.enqueueSystemEvent(text, {
|
||||
agentId: job.agentId,
|
||||
sessionKey: cronRunSessionKey,
|
||||
contextKey: `cron:${job.id}`,
|
||||
...(deliveryContext ? { deliveryContext } : {}),
|
||||
}),
|
||||
);
|
||||
if (job.wakeMode === "now" && state.deps.runHeartbeatOnce) {
|
||||
const reason = `cron:${job.id}`;
|
||||
const maxWaitMs = state.deps.wakeNowHeartbeatBusyMaxWaitMs ?? 2 * 60_000;
|
||||
@@ -1465,6 +1597,7 @@ async function executeMainSessionCronJob(
|
||||
let heartbeatResult: HeartbeatRunResult;
|
||||
for (;;) {
|
||||
if (abortSignal?.aborted) {
|
||||
removeQueuedSystemEventHandle(state, job, queuedSystemEvent);
|
||||
return { status: "error", error: timeoutErrorMessage() };
|
||||
}
|
||||
heartbeatResult = await state.deps.runHeartbeatOnce({
|
||||
@@ -1494,10 +1627,12 @@ async function executeMainSessionCronJob(
|
||||
return { status: "ok", summary: text, sessionKey: cronRunSessionKey };
|
||||
}
|
||||
if (abortSignal?.aborted) {
|
||||
removeQueuedSystemEventHandle(state, job, queuedSystemEvent);
|
||||
return { status: "error", error: timeoutErrorMessage() };
|
||||
}
|
||||
if (state.deps.nowMs() - waitStartedAt > maxWaitMs) {
|
||||
if (abortSignal?.aborted) {
|
||||
removeQueuedSystemEventHandle(state, job, queuedSystemEvent);
|
||||
return { status: "error", error: timeoutErrorMessage() };
|
||||
}
|
||||
state.deps.requestHeartbeat({
|
||||
@@ -1517,6 +1652,7 @@ async function executeMainSessionCronJob(
|
||||
return { status: "ok", summary: text, sessionKey: cronRunSessionKey };
|
||||
}
|
||||
if (heartbeatResult.status === "skipped") {
|
||||
removeQueuedSystemEventHandle(state, job, queuedSystemEvent);
|
||||
return {
|
||||
status: "skipped",
|
||||
error: heartbeatResult.reason,
|
||||
@@ -1524,6 +1660,7 @@ async function executeMainSessionCronJob(
|
||||
sessionKey: cronRunSessionKey,
|
||||
};
|
||||
}
|
||||
removeQueuedSystemEventHandle(state, job, queuedSystemEvent);
|
||||
return {
|
||||
status: "error",
|
||||
error: heartbeatResult.reason,
|
||||
@@ -1533,6 +1670,7 @@ async function executeMainSessionCronJob(
|
||||
}
|
||||
|
||||
if (abortSignal?.aborted) {
|
||||
removeQueuedSystemEventHandle(state, job, queuedSystemEvent);
|
||||
return { status: "error", error: timeoutErrorMessage() };
|
||||
}
|
||||
state.deps.requestHeartbeat({
|
||||
|
||||
@@ -9,6 +9,7 @@ import { SsrFBlockedError } from "../infra/net/ssrf.js";
|
||||
|
||||
const {
|
||||
enqueueSystemEventMock,
|
||||
consumeSelectedSystemEventEntriesMock,
|
||||
requestHeartbeatMock,
|
||||
runHeartbeatOnceMock,
|
||||
loadConfigMock,
|
||||
@@ -21,6 +22,7 @@ const {
|
||||
retireSessionMcpRuntimeMock,
|
||||
} = vi.hoisted(() => ({
|
||||
enqueueSystemEventMock: vi.fn(),
|
||||
consumeSelectedSystemEventEntriesMock: vi.fn((_sessionKey, entries) => entries ?? []),
|
||||
requestHeartbeatMock: vi.fn(),
|
||||
runHeartbeatOnceMock: vi.fn<
|
||||
(...args: unknown[]) => Promise<{ status: "ran"; durationMs: number }>
|
||||
@@ -42,8 +44,23 @@ const {
|
||||
retireSessionMcpRuntimeMock: vi.fn(async () => true),
|
||||
}));
|
||||
|
||||
function enqueueSystemEvent(...args: unknown[]) {
|
||||
return enqueueSystemEventMock(...args);
|
||||
function enqueueSystemEvent(text: string, opts?: unknown) {
|
||||
return enqueueSystemEventMock(text, opts);
|
||||
}
|
||||
|
||||
function enqueueSystemEventEntry(text: string, opts?: unknown) {
|
||||
const result = enqueueSystemEventMock(text, opts);
|
||||
if (result === false || result === null) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
text,
|
||||
ts: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function consumeSelectedSystemEventEntries(sessionKey: string, entries: readonly unknown[]) {
|
||||
return consumeSelectedSystemEventEntriesMock(sessionKey, entries);
|
||||
}
|
||||
|
||||
function requestHeartbeat(...args: unknown[]) {
|
||||
@@ -56,6 +73,8 @@ function runHeartbeatOnce(...args: unknown[]) {
|
||||
|
||||
vi.mock("../infra/system-events.js", () => ({
|
||||
enqueueSystemEvent,
|
||||
enqueueSystemEventEntry,
|
||||
consumeSelectedSystemEventEntries,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/heartbeat-wake.js", async () => {
|
||||
@@ -204,6 +223,7 @@ function expectCleanupForSessionKeys(sessionKeys: string[]) {
|
||||
describe("buildGatewayCronService", () => {
|
||||
beforeEach(() => {
|
||||
enqueueSystemEventMock.mockClear();
|
||||
consumeSelectedSystemEventEntriesMock.mockClear();
|
||||
requestHeartbeatMock.mockClear();
|
||||
runHeartbeatOnceMock.mockClear();
|
||||
loadConfigMock.mockClear();
|
||||
|
||||
@@ -32,7 +32,10 @@ import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { resolveMainScopedEventSessionKey } from "../infra/event-session-routing.js";
|
||||
import { runHeartbeatOnce } from "../infra/heartbeat-runner.js";
|
||||
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
|
||||
import { enqueueSystemEvent } from "../infra/system-events.js";
|
||||
import {
|
||||
consumeSelectedSystemEventEntries,
|
||||
enqueueSystemEventEntry,
|
||||
} from "../infra/system-events.js";
|
||||
import { getChildLogger } from "../logging.js";
|
||||
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
|
||||
import type {
|
||||
@@ -323,11 +326,17 @@ export function buildGatewayCronService(params: {
|
||||
if (!sessionKey) {
|
||||
throw new Error("Cron system event target did not resolve a session key.");
|
||||
}
|
||||
enqueueSystemEvent(text, {
|
||||
const event = enqueueSystemEventEntry(text, {
|
||||
sessionKey,
|
||||
contextKey: opts?.contextKey,
|
||||
deliveryContext: opts?.deliveryContext,
|
||||
});
|
||||
return event
|
||||
? {
|
||||
accepted: true,
|
||||
remove: () => consumeSelectedSystemEventEntries(sessionKey, [event]).length > 0,
|
||||
}
|
||||
: { accepted: false };
|
||||
},
|
||||
resolveOriginDeliveryContext: (opts) => {
|
||||
// Resolve the wake target the same way the enqueue/heartbeat deps do,
|
||||
|
||||
@@ -99,33 +99,41 @@ function findDuplicateInQueue(
|
||||
return queue.some((event) => isDuplicateSystemEvent(event, incoming));
|
||||
}
|
||||
|
||||
export function enqueueSystemEvent(text: string, options: SystemEventOptions) {
|
||||
export function enqueueSystemEventEntry(
|
||||
text: string,
|
||||
options: SystemEventOptions,
|
||||
): SystemEvent | null {
|
||||
const key = requireSessionKey(options.sessionKey);
|
||||
const entry = getOrCreateSessionQueue(key);
|
||||
// These entries are rendered as `System:` lines, so strip nested system-marker
|
||||
// spoofs at the queue boundary before any plugin/channel text reaches a prompt.
|
||||
const cleaned = sanitizeInboundSystemTags(text).trim();
|
||||
if (!cleaned) {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
const normalizedContextKey = normalizeContextKey(options.contextKey);
|
||||
const normalizedDeliveryContext = normalizeDeliveryContext(options.deliveryContext);
|
||||
if (findDuplicateInQueue(entry.queue, cleaned, normalizedContextKey, normalizedDeliveryContext)) {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
if (normalizedContextKey !== null) {
|
||||
entry.lastContextKey = normalizedContextKey;
|
||||
}
|
||||
entry.queue.push({
|
||||
const event: SystemEvent = {
|
||||
text: cleaned,
|
||||
ts: Date.now(),
|
||||
contextKey: normalizedContextKey,
|
||||
deliveryContext: normalizedDeliveryContext,
|
||||
});
|
||||
};
|
||||
entry.queue.push(event);
|
||||
if (entry.queue.length > MAX_EVENTS) {
|
||||
entry.queue.shift();
|
||||
}
|
||||
return true;
|
||||
return cloneSystemEvent(event);
|
||||
}
|
||||
|
||||
export function enqueueSystemEvent(text: string, options: SystemEventOptions) {
|
||||
return enqueueSystemEventEntry(text, options) !== null;
|
||||
}
|
||||
|
||||
export function drainSystemEventEntries(sessionKey: string): SystemEvent[] {
|
||||
|
||||
Reference in New Issue
Block a user