fix(cron): prevent invalid timestamps from stranding jobs (#121394)

* fix(cron): harden scheduling timestamp boundaries

Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe

* test(prompts): refresh cron tool snapshots

Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe

* test(cron): keep config revision fixture Date-valid

Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe

* refactor(cron): consolidate scheduling lifecycle

Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe

* refactor(cron): keep task history dependencies acyclic

Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe

* fix(cron): canonicalize timestamp auto-disable

Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe

* test(cron): verify startup overflow notifications

Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe

* chore(ci): repair main baseline gates

Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe

---------

Co-authored-by: Amp <amp@ampcode.com>
This commit is contained in:
Peter Steinberger
2026-08-10 03:20:23 -07:00
committed by GitHub
parent c6edd3e80f
commit 8e1c238c1c
88 changed files with 2510 additions and 1517 deletions
@@ -1,5 +1,6 @@
import { Value } from "typebox/value";
// Gateway Protocol tests cover cron validators behavior.
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { Value } from "typebox/value";
import { describe, expect, it } from "vitest";
import {
validateCronAddParams,
@@ -58,12 +59,21 @@ describe("cron protocol validators", () => {
consecutiveErrors: 10,
autoDisabled: {
reason: "consecutive-failures",
atMs: 2,
atMs: MAX_DATE_TIMESTAMP_MS,
consecutiveErrors: 10,
},
},
};
expect(Value.Check(CronJobSchema, job)).toBe(true);
expect(
Value.Check(CronJobSchema, {
...job,
state: {
...job.state,
autoDisabled: { ...job.state.autoDisabled, atMs: MAX_DATE_TIMESTAMP_MS + 1 },
},
}),
).toBe(false);
expect(validateCronUpdateParams(update({ state: job.state }))).toBe(false);
});
@@ -100,6 +110,32 @@ describe("cron protocol validators", () => {
]);
});
it("rejects every schedule numbers outside the ECMAScript Date range", () => {
const invalidTimestamp = MAX_DATE_TIMESTAMP_MS + 1;
expectCases(validateCronAddParams, false, [
add({ schedule: { kind: "every", everyMs: invalidTimestamp } }),
add({ schedule: { kind: "every", everyMs: 60_000, anchorMs: invalidTimestamp } }),
add({ schedule: { kind: "cron", expr: "0 * * * *", staggerMs: invalidTimestamp } }),
]);
expectCases(validateCronUpdateParams, false, [
update({ schedule: { kind: "every", everyMs: invalidTimestamp } }),
update({ schedule: { kind: "every", everyMs: 60_000, anchorMs: invalidTimestamp } }),
update({ schedule: { kind: "cron", expr: "0 * * * *", staggerMs: invalidTimestamp } }),
]);
});
it("rejects mutable scheduler state outside the ECMAScript Date range", () => {
const invalidTimestamp = MAX_DATE_TIMESTAMP_MS + 1;
expectCases(validateCronUpdateParams, false, [
update({ state: { nextRunAtMs: invalidTimestamp } }),
update({ state: { runningAtMs: invalidTimestamp } }),
update({ state: { lastRunAtMs: invalidTimestamp } }),
]);
expectCases(validateCronUpdateParams, true, [
update({ state: { nextRunAtMs: MAX_DATE_TIMESTAMP_MS } }),
]);
});
it("accepts trigger add, patch, and clear shapes", () => {
expectCases(validateCronAddParams, true, [
add({ trigger: { script: "json({ fire: true })", once: true } }),
@@ -235,6 +271,29 @@ describe("cron protocol validators", () => {
]);
});
it("accepts completion webhooks only alongside announce delivery", () => {
const completionDestination = {
mode: "webhook",
to: "https://example.invalid/complete",
} as const;
expectCases(validateCronAddParams, true, [
add({ delivery: { mode: "announce", completionDestination } }),
]);
expectCases(validateCronAddParams, false, [
add({ delivery: { mode: "none", completionDestination } }),
add({ delivery: { mode: "webhook", to: "https://example.invalid", completionDestination } }),
add({ delivery: { mode: "announce", completionDestination: null } }),
]);
expectCases(validateCronUpdateParams, true, [
update({ delivery: { completionDestination } }),
update({ delivery: { completionDestination: null } }),
]);
expectCases(validateCronUpdateParams, false, [
update({ delivery: { completionDestination: {} } }),
update({ delivery: { completionDestination: { mode: "announce", to: "https://x.test" } } }),
]);
});
it("accepts nullable delivery clears on update params", () => {
expectCases(validateCronUpdateParams, true, [
update({
+33 -25
View File
@@ -4,6 +4,14 @@ import { closedObject } from "./closed-object.js";
import { FailoverReasonSchema } from "./failover-reason.js";
import { NonEmptyString } from "./primitives.js";
// ECMAScript Date's inclusive timestamp limit. Keep public schedule numbers
// inside the range the scheduler can represent and serialize.
const MAX_DATE_TIMESTAMP_MS = 8_640_000_000_000_000;
const CronDateTimestampMsSchema = Type.Integer({
minimum: 0,
maximum: MAX_DATE_TIMESTAMP_MS,
});
/**
* Cron scheduler protocol schemas.
*
@@ -205,14 +213,14 @@ const CronScheduleSchema = Type.Union([
}),
closedObject({
kind: Type.Literal("every"),
everyMs: Type.Integer({ minimum: 1, maximum: Number.MAX_SAFE_INTEGER }),
anchorMs: Type.Optional(Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER })),
everyMs: Type.Integer({ minimum: 1, maximum: MAX_DATE_TIMESTAMP_MS }),
anchorMs: Type.Optional(Type.Integer({ minimum: 0, maximum: MAX_DATE_TIMESTAMP_MS })),
}),
closedObject({
kind: Type.Literal("cron"),
expr: NonEmptyString,
tz: Type.Optional(Type.String()),
staggerMs: Type.Optional(Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER })),
staggerMs: Type.Optional(Type.Integer({ minimum: 0, maximum: MAX_DATE_TIMESTAMP_MS })),
}),
closedObject({
// Event-driven trigger: fires once when the gateway-owned watcher running
@@ -415,16 +423,16 @@ const CronFailureNotificationDeliverySchema = closedObject({
const CronAutoDisabledSchema = closedObject({
reason: Type.Union([Type.Literal("consecutive-failures"), Type.Literal("schedule-errors")]),
atMs: Type.Integer({ minimum: 0 }),
atMs: CronDateTimestampMsSchema,
consecutiveErrors: Type.Integer({ minimum: 1 }),
});
/** Scheduler-maintained state for the latest run/delivery outcome. */
export const CronJobStateSchema = closedObject({
nextRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
scheduleActivatedAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
runningAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
nextRunAtMs: Type.Optional(CronDateTimestampMsSchema),
scheduleActivatedAtMs: Type.Optional(CronDateTimestampMsSchema),
runningAtMs: Type.Optional(CronDateTimestampMsSchema),
lastRunAtMs: Type.Optional(CronDateTimestampMsSchema),
lastRunStatus: Type.Optional(CronRunStatusSchema),
lastStatus: Type.Optional(DeprecatedCronRunStatusSchema),
lastError: Type.Optional(Type.String()),
@@ -442,10 +450,10 @@ export const CronJobStateSchema = closedObject({
lastFailureNotificationDelivered: Type.Optional(Type.Boolean()),
lastFailureNotificationDeliveryStatus: Type.Optional(CronDeliveryStatusSchema),
lastFailureNotificationDeliveryError: Type.Optional(Type.String()),
lastFailureAlertAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastTriggerEvalAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastFailureAlertAtMs: Type.Optional(CronDateTimestampMsSchema),
lastTriggerEvalAtMs: Type.Optional(CronDateTimestampMsSchema),
triggerEvalCount: Type.Optional(Type.Integer({ minimum: 0 })),
lastTriggerFireAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastTriggerFireAtMs: Type.Optional(CronDateTimestampMsSchema),
triggerState: Type.Optional(Type.Unknown()),
streamStatus: Type.Optional(
Type.Union([
@@ -466,14 +474,14 @@ export const CronJobStateSchema = closedObject({
streamSourceIdentity: Type.Optional(Type.String()),
streamDroppedBatches: Type.Optional(Type.Integer({ minimum: 0 })),
streamCoalescedBatches: Type.Optional(Type.Integer({ minimum: 0 })),
streamLastStartedAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
streamLastExitAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
streamLastStartedAtMs: Type.Optional(CronDateTimestampMsSchema),
streamLastExitAtMs: Type.Optional(CronDateTimestampMsSchema),
});
const CronJobStatePatchSchema = closedObject({
nextRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
runningAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
nextRunAtMs: Type.Optional(CronDateTimestampMsSchema),
runningAtMs: Type.Optional(CronDateTimestampMsSchema),
lastRunAtMs: Type.Optional(CronDateTimestampMsSchema),
lastRunStatus: Type.Optional(CronRunStatusSchema),
lastStatus: Type.Optional(DeprecatedCronRunStatusSchema),
lastError: Type.Optional(Type.String()),
@@ -487,10 +495,10 @@ const CronJobStatePatchSchema = closedObject({
lastFailureNotificationDelivered: Type.Optional(Type.Boolean()),
lastFailureNotificationDeliveryStatus: Type.Optional(CronDeliveryStatusSchema),
lastFailureNotificationDeliveryError: Type.Optional(Type.String()),
lastFailureAlertAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastTriggerEvalAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastFailureAlertAtMs: Type.Optional(CronDateTimestampMsSchema),
lastTriggerEvalAtMs: Type.Optional(CronDateTimestampMsSchema),
triggerEvalCount: Type.Optional(Type.Integer({ minimum: 0 })),
lastTriggerFireAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastTriggerFireAtMs: Type.Optional(CronDateTimestampMsSchema),
triggerState: Type.Optional(Type.Unknown()),
streamStatus: Type.Optional(
Type.Union([
@@ -507,8 +515,8 @@ const CronJobStatePatchSchema = closedObject({
streamRestartExhausted: Type.Optional(Type.Boolean()),
streamDroppedBatches: Type.Optional(Type.Integer({ minimum: 0 })),
streamCoalescedBatches: Type.Optional(Type.Integer({ minimum: 0 })),
streamLastStartedAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
streamLastExitAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
streamLastStartedAtMs: Type.Optional(CronDateTimestampMsSchema),
streamLastExitAtMs: Type.Optional(CronDateTimestampMsSchema),
});
/** Persisted cron job definition returned by scheduler list/get APIs. */
@@ -524,8 +532,8 @@ export const CronJobSchema = closedObject({
description: Type.Optional(Type.String()),
enabled: Type.Boolean(),
deleteAfterRun: Type.Optional(Type.Boolean()),
createdAtMs: Type.Integer({ minimum: 0 }),
updatedAtMs: Type.Integer({ minimum: 0 }),
createdAtMs: CronDateTimestampMsSchema,
updatedAtMs: CronDateTimestampMsSchema,
/** Opaque Gateway-computed token for the job definition, excluding scheduler state. */
configRevision: Type.Optional(CronConfigRevisionSchema),
schedule: CronScheduleSchema,
@@ -537,8 +545,8 @@ export const CronJobSchema = closedObject({
delivery: Type.Optional(CronDeliverySchema),
failureAlert: Type.Optional(Type.Union([Type.Literal(false), CronFailureAlertSchema])),
state: CronJobStateSchema,
nextRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
lastRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
nextRunAtMs: Type.Optional(CronDateTimestampMsSchema),
lastRunAtMs: Type.Optional(CronDateTimestampMsSchema),
lastRunStatus: Type.Optional(CronRunStatusSchema),
lastRunError: Type.Optional(Type.String()),
lastDelivered: Type.Optional(Type.Boolean()),
+31 -2
View File
@@ -1,4 +1,5 @@
/** Model-facing schema and input validation for the cron tool. */
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { Type, type TSchema } from "typebox";
import { parseCronPacingBounds } from "../../cron/pacing.js";
import type { CronPacing } from "../../cron/types.js";
@@ -100,9 +101,13 @@ function createCronScheduleSchema(): TSchema {
{
kind: optionalStringEnum(CRON_SCHEDULE_KINDS, { description: "Schedule kind" }),
at: Type.Optional(Type.String({ description: "ISO-8601 time (kind=at)" })),
everyMs: optionalPositiveIntegerSchema({ description: "Interval ms (kind=every)" }),
everyMs: optionalPositiveIntegerSchema({
description: "Interval ms (kind=every)",
maximum: MAX_DATE_TIMESTAMP_MS,
}),
anchorMs: optionalNonNegativeIntegerSchema({
description: "Start anchor ms (kind=every)",
maximum: MAX_DATE_TIMESTAMP_MS,
}),
expr: Type.Optional(
Type.String({
@@ -116,7 +121,10 @@ function createCronScheduleSchema(): TSchema {
'IANA timezone for wall-clock fields; missing=Gateway host local timezone. Example "Asia/Shanghai".',
}),
),
staggerMs: optionalNonNegativeIntegerSchema({ description: "Jitter ms (kind=cron)" }),
staggerMs: optionalNonNegativeIntegerSchema({
description: "Jitter ms (kind=cron)",
maximum: MAX_DATE_TIMESTAMP_MS,
}),
command: Type.Optional(
Type.Array(Type.String({ minLength: 1 }), {
minItems: 1,
@@ -198,6 +206,19 @@ function cronDeliverySchema(params: { nullableClears: boolean }) {
},
{ additionalProperties: true },
);
const completionDestinationObject = Type.Object(
{
mode: Type.Literal("webhook"),
to: Type.String({
minLength: 1,
description: "Completion webhook target; only valid with delivery.mode=announce",
}),
},
{
additionalProperties: true,
description: "Additional completion webhook; requires delivery.mode=announce",
},
);
return Type.Optional(
Type.Object(
@@ -224,6 +245,14 @@ function cronDeliverySchema(params: { nullableClears: boolean }) {
}),
)
: Type.Optional(failureDestinationObject),
completionDestination: params.nullableClears
? Type.Optional(
Type.Union([completionDestinationObject, Type.Null()], {
description:
"Completion webhook destination; requires delivery.mode=announce; null clears.",
}),
)
: Type.Optional(completionDestinationObject),
},
{ additionalProperties: true },
),
+99 -10
View File
@@ -2,6 +2,7 @@ import {
findLlamacppGbnfSchemaViolations,
normalizeToolParameterSchema,
} from "@openclaw/ai/internal/openai";
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
// Cron tool schema tests cover the provider-facing parameter shape and runtime
// validation compatibility for cron jobs.
import { Value } from "typebox/value";
@@ -205,16 +206,27 @@ describe("createCronToolSchema", () => {
it("advertises numeric cron params with runtime bounds", () => {
for (const path of ["job.schedule.everyMs", "patch.schedule.everyMs"]) {
expect(propertyAt(schemaRecord, path)).toMatchObject({ type: "integer", minimum: 1 });
expect(propertyAt(schemaRecord, path)).toMatchObject({
type: "integer",
minimum: 1,
maximum: MAX_DATE_TIMESTAMP_MS,
});
}
for (const path of [
"job.schedule.anchorMs",
"job.schedule.staggerMs",
"patch.schedule.anchorMs",
"patch.schedule.staggerMs",
"job.failureAlert.cooldownMs",
"patch.failureAlert.cooldownMs",
]) {
for (const path of ["job.schedule.anchorMs", "patch.schedule.anchorMs"]) {
expect(propertyAt(schemaRecord, path)).toMatchObject({
type: "integer",
minimum: 0,
maximum: MAX_DATE_TIMESTAMP_MS,
});
}
for (const path of ["job.schedule.staggerMs", "patch.schedule.staggerMs"]) {
expect(propertyAt(schemaRecord, path)).toMatchObject({
type: "integer",
minimum: 0,
maximum: MAX_DATE_TIMESTAMP_MS,
});
}
for (const path of ["job.failureAlert.cooldownMs", "patch.failureAlert.cooldownMs"]) {
expect(propertyAt(schemaRecord, path)).toMatchObject({ type: "integer", minimum: 0 });
}
for (const path of ["job.failureAlert.after", "patch.failureAlert.after"]) {
@@ -247,18 +259,95 @@ describe("createCronToolSchema", () => {
}
});
it("job.delivery exposes mode, channel, to, threadId, bestEffort, accountId, failureDestination", () => {
it("job.delivery exposes all supported delivery destinations", () => {
expect(keysAt(schemaRecord, "job.delivery")).toEqual(
[
"accountId",
"bestEffort",
"channel",
"completionDestination",
"failureDestination",
"mode",
"threadId",
"to",
].toSorted(),
);
const jobCompletion = propertyAt(schemaRecord, "job.delivery.completionDestination");
expect(keysAt(schemaRecord, "job.delivery.completionDestination")).toEqual(["mode", "to"]);
expect(jobCompletion?.required).toEqual(["mode", "to"]);
expect(propertyAt(schemaRecord, "job.delivery.completionDestination.to")).toMatchObject({
type: "string",
minLength: 1,
});
const patchCompletion = propertyAt(schemaRecord, "patch.delivery.completionDestination");
const patchCompletionObject = (
patchCompletion?.anyOf as Array<Record<string, unknown>> | undefined
)?.find((entry) => entry.type === "object");
expect(
Object.keys(
(patchCompletionObject?.properties as Record<string, unknown> | undefined) ?? {},
).toSorted(),
).toEqual(["mode", "to"]);
expect(
Value.Check(schema, {
action: "add",
job: {
schedule: { kind: "every", everyMs: 60_000 },
payload: { kind: "agentTurn", message: "run" },
delivery: {
mode: "announce",
completionDestination: { mode: "webhook", to: "https://example.invalid/done" },
},
},
}),
).toBe(true);
expect(
Value.Check(schema, {
action: "update",
id: "job-1",
patch: { delivery: { completionDestination: null } },
}),
).toBe(true);
for (const completionDestination of [
null,
{},
{ mode: "webhook" },
{ to: "https://example.invalid/done" },
{ mode: "announce", to: "https://example.invalid/done" },
{ mode: "webhook", to: "" },
"https://example.invalid/done",
]) {
expect(
Value.Check(schema, {
action: "add",
job: {
schedule: { kind: "every", everyMs: 60_000 },
payload: { kind: "agentTurn", message: "run" },
delivery: { mode: "announce", completionDestination },
},
}),
).toBe(false);
}
for (const providerSchema of [
providerSchemaRecord,
jjccGeminiSchemaRecord,
llamacppSchemaRecord,
]) {
expect(propertyAt(providerSchema, "job.delivery.completionDestination")).toMatchObject({
type: "object",
required: ["mode", "to"],
});
}
for (const providerSchema of [providerSchemaRecord, jjccGeminiSchemaRecord]) {
expect(propertyAt(providerSchema, "patch.delivery.completionDestination")).toMatchObject({
type: "object",
required: ["mode", "to"],
description: expect.stringContaining("null clears"),
});
}
expect(
propertyAt(llamacppSchemaRecord, "patch.delivery.completionDestination")?.anyOf,
).toContainEqual({ type: "null" });
});
it("job.payload exposes conversational and script payload fields", () => {
+17
View File
@@ -1,5 +1,6 @@
// Cron tool tests cover schedule guidance, scoped job operations, delivery
// context inheritance, session routing, and agent id ownership.
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { beforeEach, describe, expect, it, vi } from "vitest";
const { callGatewayMock, extractDeliveryInfoMock } = vi.hoisted(() => ({
@@ -1103,6 +1104,22 @@ describe("cron tool", () => {
});
});
it("canonicalizes the inclusive Date maximum without losing service compatibility", async () => {
const tool = createTestCronTool();
await tool.execute("call-date-max", {
action: "add",
job: {
name: "far-future",
schedule: { atMs: MAX_DATE_TIMESTAMP_MS },
payload: { kind: "systemEvent", text: "hello" },
},
});
expect(expectSingleGatewayCallMethod("cron.add")).toMatchObject({
schedule: { kind: "at", at: new Date(MAX_DATE_TIMESTAMP_MS).toISOString() },
});
});
it("preserves omitted declaration enablement and forwards explicit enablement", async () => {
const tool = createTestCronTool();
const baseJob = {
+1 -1
View File
@@ -196,7 +196,7 @@ PACED LOOP: recurring job + pacing{min?,max?} durations ("15m","4h"; at least on
TRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call("exec",{command:"..."}).
DELIVERY {mode:"none"|"announce"|"webhook",channel?,to?,threadId?,bestEffort?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:"none". webhook posts finished-run event to URL in \`to\`.
DELIVERY {mode:"none"|"announce"|"webhook",channel?,to?,threadId?,bestEffort?,completionDestination?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:"none". webhook posts finished-run event to URL in \`to\`. To keep announce delivery and also POST completion, use mode:"announce" with completionDestination:{mode:"webhook",to:"https://..."}.
Job wakeMode (main jobs): "now"(default)|"next-heartbeat". Restricted automation-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.`,
parameters: createCronToolSchema(),
@@ -12,7 +12,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
import { heartbeatTaskDeclarationKey, isHeartbeatTaskCronJob } from "../cron/heartbeat-task.js";
import { cronSchedulingInputsEqual } from "../cron/schedule-identity.js";
import { readHeartbeatMonitorScratch } from "../cron/scratch-store.js";
import { computeJobNextRunAtMs, hasScheduledNextRunAtMs } from "../cron/service/jobs.js";
import { computeJobNextRunAtMs, hasScheduledNextRunAtMs } from "../cron/service/jobs-scheduling.js";
import { resolveCronJobsStorePathFromConfig } from "../cron/store.js";
import { cronStoreKey } from "../cron/store/key.js";
import {
+1 -1
View File
@@ -9,7 +9,7 @@ import {
} from "../../../../packages/normalization-core/src/string-coerce.js";
import { parseAbsoluteTimeMs } from "../../../cron/parse.js";
import { getInvalidPersistedCronJobReason } from "../../../cron/persisted-shape.js";
import { coerceFiniteScheduleNumber } from "../../../cron/schedule.js";
import { coerceFiniteScheduleNumber } from "../../../cron/schedule-number.js";
import { inferCronJobName } from "../../../cron/service/normalize.js";
import { normalizeCronStaggerMs, resolveDefaultCronStaggerMs } from "../../../cron/stagger.js";
import {
+2 -1
View File
@@ -1,3 +1,4 @@
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it } from "vitest";
import { resolveCronJobConfigRevision } from "./config-revision.js";
import { setupCronServiceSuite } from "./service.test-harness.js";
@@ -152,7 +153,7 @@ describe("resolveCronJobConfigRevision", () => {
{
...makeJob(),
id: "command-empty-env",
schedule: { kind: "every", everyMs: Number.MAX_SAFE_INTEGER, anchorMs: 0 },
schedule: { kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS, anchorMs: 0 },
payload: { kind: "command", argv: ["true"], env: {}, input: "" },
failureAlert: false,
},
@@ -8,10 +8,6 @@ const channelPluginRuntimeLoader = createLazyImportLoader<ChannelPluginRuntime>(
() => import("../../channels/plugins/index.js"),
);
async function loadChannelPluginRuntime() {
return await channelPluginRuntimeLoader.load();
}
/** Resolves channel-specific cron output preferences from loaded channel plugins. */
export async function resolveCronChannelOutputPolicy(
channel: string | undefined,
@@ -23,7 +19,7 @@ export async function resolveCronChannelOutputPolicy(
if (!channelId) {
return { preferFinalAssistantVisibleText: opts?.deliveryRequested !== true };
}
const { getChannelPlugin } = await loadChannelPluginRuntime();
const { getChannelPlugin } = await channelPluginRuntimeLoader.load();
return {
preferFinalAssistantVisibleText:
getChannelPlugin(channelId)?.outbound?.preferFinalAssistantVisibleText === true,
@@ -43,7 +39,7 @@ export async function resolveCurrentChannelTarget(params: {
if (!channelId) {
return params.to;
}
const { getChannelPlugin } = await loadChannelPluginRuntime();
const { getChannelPlugin } = await channelPluginRuntimeLoader.load();
return (
getChannelPlugin(channelId)?.threading?.resolveCurrentChannelId?.({
to: params.to,
@@ -55,24 +55,6 @@ const outboundSessionRuntimeLoader = createLazyImportLoader(
const transcriptRuntimeLoader = createLazyImportLoader(
() => import("../../config/sessions/transcript.runtime.js"),
);
async function loadDeliveryOutboundRuntime(): Promise<
typeof import("./delivery-outbound.runtime.js")
> {
return await deliveryOutboundRuntimeLoader.load();
}
async function loadOutboundSessionRuntime(): Promise<
typeof import("../../infra/outbound/outbound-session.js")
> {
return await outboundSessionRuntimeLoader.load();
}
async function loadTranscriptRuntime(): Promise<
typeof import("../../config/sessions/transcript.runtime.js")
> {
return await transcriptRuntimeLoader.load();
}
export function shouldQueueCronAwareness(params: {
job: CronJob;
delivery: SuccessfulDeliveryTarget;
@@ -191,7 +173,7 @@ export async function queueCronAwarenessSystemEvent(params: {
targetText?: string;
}): Promise<void> {
try {
const { enqueueSystemEvent } = await loadDeliveryOutboundRuntime();
const { enqueueSystemEvent } = await deliveryOutboundRuntimeLoader.load();
const mainSessionKey = resolveCronAwarenessMainSessionKey({
cfg: params.cfg,
agentId: params.agentId,
@@ -344,7 +326,7 @@ async function resolveCronDeliveryRouteSessionKey(params: {
}): Promise<string> {
try {
const { resolveOutboundSessionRoute, ensureOutboundSessionEntry } =
await loadOutboundSessionRuntime();
await outboundSessionRuntimeLoader.load();
const route = await resolveOutboundSessionRoute({
cfg: params.cfg,
channel: params.delivery.channel,
@@ -534,7 +516,7 @@ async function appendDirectCronDeliveryTranscriptMirror(params: {
return;
}
try {
const { appendAssistantMessageToSessionTranscript } = await loadTranscriptRuntime();
const { appendAssistantMessageToSessionTranscript } = await transcriptRuntimeLoader.load();
const result = await appendAssistantMessageToSessionTranscript(params.mirror);
if (!result.ok) {
await logCronDeliveryWarn(
@@ -24,10 +24,9 @@ import { stringifyRouteThreadId } from "../../plugin-sdk/channel-route.js";
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
import { shouldAttemptTtsPayload } from "../../tts/tts-config.js";
import { createCronExecutionId } from "../run-id.js";
import { hasScheduledNextRunAtMs } from "../service/jobs.js";
import { hasScheduledNextRunAtMs } from "../service/jobs-scheduling.js";
import type { CronJob } from "../types.js";
import type { DeliveryTargetResolution } from "./delivery-target.js";
import { cleanupCronRunSessionAfterRun } from "./session-cleanup.js";
type SuccessfulDeliveryTarget = Extract<DeliveryTargetResolution, { ok: true }>;
@@ -37,27 +36,6 @@ export const DIRECT_CRON_DELIVERY_COMPLETION_RETENTION = {
maxEntries: 2_000,
} as const satisfies DeliveryQueueCompletionRetention;
/** Deletes or retires ephemeral direct-delivery cron sessions for delete-after-run jobs. */
export async function cleanupDirectCronSession(params: {
job: CronJob;
agentSessionKey: string;
sessionId: string;
lifecycleRevision: string;
sessionUpdatedAt: number;
beforeSessionDelete?: () => void;
retireReason: string;
}): Promise<void> {
await cleanupCronRunSessionAfterRun({
job: params.job,
agentSessionKey: params.agentSessionKey,
sessionId: params.sessionId,
lifecycleRevision: params.lifecycleRevision,
sessionUpdatedAt: params.sessionUpdatedAt,
beforeDelete: params.beforeSessionDelete,
reason: params.retireReason,
});
}
export function normalizeDeliveryTarget(channel: string, to: string): string {
const toTrimmed = to.trim();
return normalizeTargetForProvider(channel, toTrimmed) ?? toTrimmed;
@@ -124,14 +102,6 @@ const deliverySubagentRegistryRuntimeLoader = createLazyImportLoader(
() => import("./delivery-subagent-registry.runtime.js"),
);
async function loadDeliveryLoggerRuntime(): Promise<typeof import("./delivery-logger.runtime.js")> {
return await deliveryLoggerRuntimeLoader.load();
}
async function loadTtsRuntime(): Promise<typeof import("../../tts/tts.runtime.js")> {
return await ttsRuntimeLoader.load();
}
export async function loadDeliverySubagentRegistryRuntime(): Promise<
typeof import("./delivery-subagent-registry.runtime.js")
> {
@@ -139,17 +109,17 @@ export async function loadDeliverySubagentRegistryRuntime(): Promise<
}
export async function logCronDeliveryWarn(message: string): Promise<void> {
const { logWarn } = await loadDeliveryLoggerRuntime();
const { logWarn } = await deliveryLoggerRuntimeLoader.load();
logWarn(message);
}
export async function logCronDeliveryError(message: string): Promise<void> {
const { logError } = await loadDeliveryLoggerRuntime();
const { logError } = await deliveryLoggerRuntimeLoader.load();
logError(message);
}
export function logCronDeliveryErrorDeferred(message: string): void {
void loadDeliveryLoggerRuntime().then(({ logError }) => {
void deliveryLoggerRuntimeLoader.load().then(({ logError }) => {
logError(message);
});
}
@@ -191,7 +161,7 @@ export async function maybeApplyTtsToCronPayloads(params: {
) {
return params.payloads;
}
const { maybeApplyTtsToPayload } = await loadTtsRuntime();
const { maybeApplyTtsToPayload } = await ttsRuntimeLoader.load();
return await Promise.all(
params.payloads.map((payload) =>
maybeApplyTtsToPayload({
+4 -21
View File
@@ -35,7 +35,6 @@ import {
} from "./delivery-dispatch-awareness.js";
import {
buildDirectCronDeliveryIdempotencyKey,
cleanupDirectCronSession,
DIRECT_CRON_DELIVERY_COMPLETION_RETENTION,
isCompletedDirectCronDelivery,
isStaleCronDelivery,
@@ -70,23 +69,7 @@ const deliveryOutboundRuntimeLoader = createLazyImportLoader(
const subagentFollowupRuntimeLoader = createLazyImportLoader(
() => import("./subagent-followup.runtime.js"),
);
async function loadDeliveryOutboundRuntime(): Promise<
typeof import("./delivery-outbound.runtime.js")
> {
return await deliveryOutboundRuntimeLoader.load();
}
async function loadSubagentFollowupRuntime(): Promise<
typeof import("./subagent-followup.runtime.js")
> {
return await subagentFollowupRuntimeLoader.load();
}
export {
cleanupDirectCronSession,
queueCronMessageToolDeliveryAwareness,
resolveCronDeliveryBestEffort,
};
export { queueCronMessageToolDeliveryAwareness, resolveCronDeliveryBestEffort };
/** Dispatches cron run output through verified message-tool or direct delivery paths. */
export async function dispatchCronDelivery(
params: DispatchCronDeliveryParams,
@@ -201,7 +184,7 @@ export async function dispatchCronDelivery(
createOutboundSendDeps,
resolveAgentOutboundIdentity,
sendDurableMessageBatch,
} = await loadDeliveryOutboundRuntime();
} = await deliveryOutboundRuntimeLoader.load();
const identity = resolveAgentOutboundIdentity(params.cfgWithAgentDefaults, params.agentId);
try {
const summaryFallbackText = resolveDirectCronSummaryFallbackText({
@@ -590,7 +573,7 @@ export async function dispatchCronDelivery(
const needsSubagentFollowupRuntime =
shouldCheckCompletedDescendants || activeSubagentRuns > 0 || expectedSubagentFollowup;
const subagentFollowupRuntime = needsSubagentFollowupRuntime
? await loadSubagentFollowupRuntime()
? await subagentFollowupRuntimeLoader.load()
: undefined;
// Also check for already-completed descendants. If the subagent finished
// before delivery-dispatch runs, activeSubagentRuns is 0 and
@@ -712,7 +695,7 @@ export async function dispatchCronDelivery(
// inherited shared-bucket target was refused). We never send here, so a
// deleteAfterRun cron must still retire its session/transcript before
// returning — otherwise the one-shot session leaks. Safe no-op for
// non-deleteAfterRun / non-cron sessions (see cleanupDirectCronSession).
// Cleanup is a no-op for non-deleteAfterRun or non-cron sessions.
await cleanupDirectCronSessionIfNeeded();
if (!params.deliveryBestEffort) {
return buildDeliveryState(failDeliveryTarget(params.resolvedDelivery.error.message));
+3 -15
View File
@@ -45,10 +45,6 @@ const targetsRuntimeLoader = createLazyImportLoader(
() => import("../../infra/outbound/targets.runtime.js"),
);
async function loadTargetsRuntime() {
return await targetsRuntimeLoader.load();
}
async function resolveOutboundTargetWithRuntime(
params: Parameters<typeof tryResolveLoadedOutboundTarget>[0],
) {
@@ -57,7 +53,7 @@ async function resolveOutboundTargetWithRuntime(
if (loaded) {
return loaded;
}
const { resolveOutboundTarget } = await loadTargetsRuntime();
const { resolveOutboundTarget } = await targetsRuntimeLoader.load();
return resolveOutboundTarget({ ...params, allowBootstrap: true });
} catch (err) {
return {
@@ -74,14 +70,6 @@ const deliveryTargetRuntimeLoader = createLazyImportLoader(
() => import("./delivery-target.runtime.js"),
);
async function loadChannelSelectionRuntime() {
return await channelSelectionRuntimeLoader.load();
}
async function loadDeliveryTargetRuntime() {
return await deliveryTargetRuntimeLoader.load();
}
function isNonEmptyThreadId(value: string | number | undefined | null): value is string | number {
return value != null && value !== "";
}
@@ -146,7 +134,7 @@ export async function resolveDeliveryTarget(
const requestedChannel = typeof jobPayload.channel === "string" ? jobPayload.channel : "last";
const explicitTo = typeof jobPayload.to === "string" ? jobPayload.to : undefined;
const allowMismatchedLastTo = requestedChannel === "last";
const deliveryTargetRuntime = await loadDeliveryTargetRuntime();
const deliveryTargetRuntime = await deliveryTargetRuntimeLoader.load();
const sessionCfg = cfg.session;
const mainSessionKey = resolveAgentMainSessionKey({ cfg, agentId });
@@ -199,7 +187,7 @@ export async function resolveDeliveryTarget(
fallbackChannel = preliminary.lastChannel;
} else {
try {
const { resolveMessageChannelSelection } = await loadChannelSelectionRuntime();
const { resolveMessageChannelSelection } = await channelSelectionRuntimeLoader.load();
const selection = await resolveMessageChannelSelection({ cfg });
fallbackChannel = selection.channel;
} catch (err) {
+2 -10
View File
@@ -46,14 +46,6 @@ async function loadCodexNativeWebSearch() {
return await codexNativeWebSearchLoader.load();
}
async function loadWebToolRuntimeContext() {
return await webToolRuntimeContextLoader.load();
}
async function loadWebSearchRuntime() {
return await webSearchRuntimeLoader.load();
}
type CronDeliveryRuntime = typeof import("./run-delivery.runtime.js");
export type ResolvedCronDeliveryTarget = Awaited<
ReturnType<CronDeliveryRuntime["resolveDeliveryTarget"]>
@@ -213,14 +205,14 @@ export async function createCronToolsAllowPreflightDiagnostics(params: {
) {
return undefined;
}
const { resolveWebSearchToolRuntimeContext } = await loadWebToolRuntimeContext();
const { resolveWebSearchToolRuntimeContext } = await webToolRuntimeContextLoader.load();
const { config, preferRuntimeProviders, runtimeWebSearch } = resolveWebSearchToolRuntimeContext(
{
config: params.cfg,
lateBindRuntimeConfig: true,
},
);
const { hasUsableWebSearchProvider } = await loadWebSearchRuntime();
const { hasUsableWebSearchProvider } = await webSearchRuntimeLoader.load();
const hasWebSearchProvider = hasUsableWebSearchProvider({
config,
agentDir: params.agentDir,
@@ -1,7 +1,6 @@
// Runtime delivery seam for isolated cron agent run orchestration.
export { resolveDeliveryTarget } from "./delivery-target.js";
export {
cleanupDirectCronSession,
dispatchCronDelivery,
queueCronMessageToolDeliveryAwareness,
resolveCronDeliveryBestEffort,
@@ -23,15 +23,11 @@ const cronExecutionCliRuntimeLoader = createLazyImportLoader(
() => import("./run-execution-cli.runtime.js"),
);
async function loadCronExecutionCliRuntime() {
return await cronExecutionCliRuntimeLoader.load();
}
/** Lazily resolves complete CLI bindings so cron continuations preserve reuse metadata. */
export async function getCliSessionBinding(
...args: Parameters<typeof import("../../agents/cli-session.js").getCliSessionBinding>
): Promise<ReturnType<typeof import("../../agents/cli-session.js").getCliSessionBinding>> {
const runtime = await loadCronExecutionCliRuntime();
const runtime = await cronExecutionCliRuntimeLoader.load();
return runtime.getCliSessionBinding(...args);
}
@@ -39,6 +35,6 @@ export async function getCliSessionBinding(
export async function runCliAgent(
...args: Parameters<typeof import("../../agents/cli-runner.js").runCliAgent>
): ReturnType<typeof import("../../agents/cli-runner.js").runCliAgent> {
const runtime = await loadCronExecutionCliRuntime();
const runtime = await cronExecutionCliRuntimeLoader.load();
return runtime.runCliAgent(...args);
}
+2 -10
View File
@@ -79,14 +79,6 @@ const cronSubagentRegistryRuntimeLoader = createLazyImportLoader<CronSubagentReg
() => import("./run-subagent-registry.runtime.js"),
);
async function loadCronEmbeddedRuntime() {
return await cronEmbeddedRuntimeLoader.load();
}
async function loadCronSubagentRegistryRuntime() {
return await cronSubagentRegistryRuntimeLoader.load();
}
function hasCliSessionReuseMetadata(binding: CliSessionBinding): boolean {
return Object.entries(binding).some(([key, value]) => key !== "sessionId" && value !== undefined);
}
@@ -582,7 +574,7 @@ function createCronPromptExecutor(params: {
acceptedContextEngineTurnCandidate = contextEngineTurnCandidate;
return result;
}
const { resolveFastModeState, runEmbeddedAgent } = await loadCronEmbeddedRuntime();
const { resolveFastModeState, runEmbeddedAgent } = await cronEmbeddedRuntimeLoader.load();
const promptCacheKey = resolveIsolatedCronPromptCacheKey({
job: params.job,
agentId: params.agentId,
@@ -929,7 +921,7 @@ export async function executeCronRun(params: {
let hasActiveDescendants = false;
if (shouldRetryInterimAck) {
const { countActiveDescendantRuns, listDescendantRunsForRequester } =
await loadCronSubagentRegistryRuntime();
await cronSubagentRegistryRuntimeLoader.load();
hasFreshDescendants = listDescendantRunsForRequester(params.runSessionKey).some((entry) => {
const descendantStartedAt =
typeof entry.execution.startedAt === "number"
+21 -39
View File
@@ -32,28 +32,17 @@ import {
setSessionRuntimeModel,
} from "./run.runtime.js";
import type { RunCronAgentTurnResult } from "./run.types.js";
import { cleanupCronRunSessionAfterRun } from "./session-cleanup.js";
type CronExecutionRuntime = typeof import("./run-executor.runtime.js");
type CronExecutionResult = Awaited<ReturnType<CronExecutionRuntime["executeCronRun"]>>;
const cronContextRuntimeLoader = createLazyImportLoader(() => import("./run-context.runtime.js"));
async function loadCronContextRuntime() {
return await cronContextRuntimeLoader.load();
}
function resolvePositiveContextTokens(value: unknown): number | undefined {
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined;
}
async function loadCliRunnerRuntime() {
return await import("../../agents/cli-runner.runtime.js");
}
async function loadUsageFormatRuntime() {
return await import("../../utils/usage-format.js");
}
export async function finalizeCronRun(params: {
prepared: PreparedCronRunContext;
execution: CronExecutionResult;
@@ -66,6 +55,18 @@ export async function finalizeCronRun(params: {
const finalRunResult = execution.runResult;
const payloads = finalRunResult.payloads ?? [];
let telemetry: CronRunTelemetry | undefined;
const cleanupRunSession = async (reason: string) => {
await cleanupCronRunSessionAfterRun({
job: prepared.input.job,
agentSessionKey: prepared.agentSessionKey,
sessionId: prepared.currentRunSessionId(),
lifecycleRevision: prepared.cronSession.lifecycleRevision,
sessionUpdatedAt: prepared.cronSession.sessionEntry.updatedAt,
beforeDelete: params.beforeSessionDelete,
reason,
});
params.markCronRunSessionCleanupAttempted();
};
// Late aborted results may still contain billable usage. Recheck before each
// metadata mutation because lazy runtime loads below can yield to the timeout.
@@ -93,7 +94,7 @@ export async function finalizeCronRun(params: {
execution.liveSelection.provider;
const contextTokens =
resolvePositiveContextTokens(prepared.agentCfg?.contextTokens) ??
(await loadCronContextRuntime()).lookupContextTokens(modelUsed, {
(await cronContextRuntimeLoader.load()).lookupContextTokens(modelUsed, {
allowAsyncLoad: false,
}) ??
resolvePositiveContextTokens(prepared.cronSession.sessionEntry.contextTokens) ??
@@ -109,19 +110,20 @@ export async function finalizeCronRun(params: {
const cliSessionBinding = finalRunResult.meta?.agentMeta?.cliSessionBinding;
const cliSessionId = finalRunResult.meta?.agentMeta?.sessionId?.trim();
if (finalRunResult.meta?.agentMeta?.clearCliSessionBinding === true) {
const { clearCliSession } = await loadCliRunnerRuntime();
const { clearCliSession } = await import("../../agents/cli-runner.runtime.js");
clearCliSession(prepared.cronSession.sessionEntry, providerUsed);
} else if (cliSessionBinding?.sessionId?.trim()) {
const { setCliSessionBinding } = await loadCliRunnerRuntime();
const { setCliSessionBinding } = await import("../../agents/cli-runner.runtime.js");
setCliSessionBinding(prepared.cronSession.sessionEntry, providerUsed, cliSessionBinding);
} else if (cliSessionId) {
const { setCliSessionId } = await loadCliRunnerRuntime();
const { setCliSessionId } = await import("../../agents/cli-runner.runtime.js");
setCliSessionId(prepared.cronSession.sessionEntry, providerUsed, cliSessionId);
}
}
}
if (hasNonzeroUsage(usage)) {
const { estimateUsageCost, resolveModelCostConfig } = await loadUsageFormatRuntime();
const { estimateUsageCost, resolveModelCostConfig } =
await import("../../utils/usage-format.js");
const input = usage.input ?? 0;
const output = usage.output ?? 0;
const cacheRead = usage.cacheRead ?? 0;
@@ -271,17 +273,7 @@ export async function finalizeCronRun(params: {
if (finalRunResult.meta?.aborted === true && !cronPayloadOutcome.hasFatalErrorPayload) {
const metaErrorMessage = normalizeOptionalString(finalRunResult.meta.error?.message);
const error = metaErrorMessage ?? "cron isolated agent run aborted";
const { cleanupDirectCronSession } = await loadCronDeliveryRuntime();
await cleanupDirectCronSession({
job: prepared.input.job,
agentSessionKey: prepared.agentSessionKey,
sessionId: prepared.currentRunSessionId(),
lifecycleRevision: prepared.cronSession.lifecycleRevision,
sessionUpdatedAt: prepared.cronSession.sessionEntry.updatedAt,
beforeSessionDelete: params.beforeSessionDelete,
retireReason: "cron-delete-after-run-aborted",
});
params.markCronRunSessionCleanupAttempted();
await cleanupRunSession("cron-delete-after-run-aborted");
return prepared.withRunSession({
status: "error",
error,
@@ -421,17 +413,7 @@ export async function finalizeCronRun(params: {
if (hasFatalStructuredErrorPayload && prepared.deliveryRequested) {
// Structured run error payloads belong in cron state and failure alerts,
// not the normal completion announce path where provider JSON can leak.
const { cleanupDirectCronSession } = await loadCronDeliveryRuntime();
await cleanupDirectCronSession({
job: prepared.input.job,
agentSessionKey: prepared.agentSessionKey,
sessionId: prepared.currentRunSessionId(),
lifecycleRevision: prepared.cronSession.lifecycleRevision,
sessionUpdatedAt: prepared.cronSession.sessionEntry.updatedAt,
beforeSessionDelete: params.beforeSessionDelete,
retireReason: "cron-delete-after-run-fatal-error",
});
params.markCronRunSessionCleanupAttempted();
await cleanupRunSession("cron-delete-after-run-fatal-error");
const deliveryTrace = buildCronDeliveryTrace({
deliveryPlan: prepared.deliveryPlan,
resolvedDelivery: prepared.resolvedDelivery,
@@ -8,8 +8,8 @@ import type { CronDeliveryMode } from "../types.js";
import type { MutableCronSession } from "./run-session-state.js";
import {
buildSafeExternalPromptMock,
callGatewayMock,
clearFastTestEnv,
cleanupDirectCronSessionMock,
dispatchCronDeliveryMock,
getChannelPluginMock,
isCliProviderMock,
@@ -1224,15 +1224,7 @@ describe("runCronIsolatedAgentTurn message tool policy", () => {
expect(result.delivered).toBe(false);
expect(result.deliveryAttempted).toBe(false);
expect(dispatchCronDeliveryMock).not.toHaveBeenCalled();
expect(cleanupDirectCronSessionMock).toHaveBeenCalledWith({
job: expect.objectContaining({ id: "fatal-error-payload" }),
agentSessionKey: "agent:default:cron:message-tool-policy",
sessionId: "test-session-id",
lifecycleRevision: "test-lifecycle-revision",
sessionUpdatedAt: expect.any(Number),
beforeSessionDelete: expect.any(Function),
retireReason: "cron-delete-after-run-fatal-error",
});
expect(callGatewayMock).not.toHaveBeenCalled();
expectDeliveryFields(result.delivery, {
intended: { channel: "messagechat", to: "123", source: "explicit" },
resolved: { ok: true, channel: "messagechat", to: "123", source: "explicit" },
@@ -1260,18 +1252,7 @@ describe("runCronIsolatedAgentTurn message tool policy", () => {
});
expect(dispatchCronDeliveryMock).not.toHaveBeenCalled();
expect(cleanupDirectCronSessionMock).toHaveBeenCalledWith({
job: expect.objectContaining({
id: "fatal-delete-after-run",
deleteAfterRun: true,
}),
agentSessionKey: "agent:default:cron:message-tool-policy",
sessionId: "test-session-id",
lifecycleRevision: "test-lifecycle-revision",
sessionUpdatedAt: expect.any(Number),
beforeSessionDelete: expect.any(Function),
retireReason: "cron-delete-after-run-fatal-error",
});
expect(callGatewayMock).toHaveBeenCalledTimes(1);
});
it("skips cron fallback delivery when the message tool already sent to the same target", async () => {
@@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";
import { makeIsolatedAgentJobFixture, makeIsolatedAgentParamsFixture } from "./job-fixtures.js";
import { setupRunCronIsolatedAgentTurnSuite } from "./run.suite-helpers.js";
import {
cleanupDirectCronSessionMock,
callGatewayMock,
dispatchCronDeliveryMock,
loadRunCronIsolatedAgentTurn,
resolveCronDeliveryPlanMock,
@@ -129,15 +129,7 @@ describe("runCronIsolatedAgentTurn - meta.error status propagation", () => {
expect(result.status).toBe("error");
expect(result.error).toBe("cron isolated agent run aborted");
expect(cleanupDirectCronSessionMock).toHaveBeenCalledWith({
job: expect.objectContaining({ deleteAfterRun: true }),
agentSessionKey: "agent:default:cron:test",
sessionId: "test-session-id",
lifecycleRevision: "test-lifecycle-revision",
sessionUpdatedAt: expect.any(Number),
beforeSessionDelete: expect.any(Function),
retireReason: "cron-delete-after-run-aborted",
});
expect(callGatewayMock).toHaveBeenCalledTimes(1);
});
it("marks a completed embedded run with no final payload as a cron error", async () => {
@@ -90,7 +90,6 @@ export const resolveCronDeliveryPlanMock = createMock();
export const resolveDeliveryTargetMock = createMock();
export const dispatchCronDeliveryMock = createMock();
export const queueCronMessageToolDeliveryAwarenessMock = createMock();
export const cleanupDirectCronSessionMock = createMock();
export const preflightCronModelProviderMock = createMock();
export const resolveSessionAuthProfileOverrideMock = createMock();
export const resolveFastModeStateMock = createMock();
@@ -397,7 +396,6 @@ vi.mock("./run-delivery.runtime.js", async () => {
);
return {
...actual,
cleanupDirectCronSession: cleanupDirectCronSessionMock,
resolveDeliveryTarget: resolveDeliveryTargetMock,
dispatchCronDelivery: dispatchCronDeliveryMock,
queueCronMessageToolDeliveryAwareness: queueCronMessageToolDeliveryAwarenessMock,
@@ -760,8 +758,6 @@ function resetRunOutcomeMocks(): void {
);
queueCronMessageToolDeliveryAwarenessMock.mockReset();
queueCronMessageToolDeliveryAwarenessMock.mockResolvedValue(undefined);
cleanupDirectCronSessionMock.mockReset();
cleanupDirectCronSessionMock.mockResolvedValue(undefined);
preflightCronModelProviderMock.mockReset();
preflightCronModelProviderMock.mockResolvedValue({ status: "available" });
resolveSessionAuthProfileOverrideMock.mockReset();
+1 -5
View File
@@ -40,10 +40,6 @@ import { cleanupCronRunSessionAfterRun } from "./session-cleanup.js";
const cronExecutorRuntimeLoader = createLazyImportLoader(() => import("./run-executor.runtime.js"));
async function loadCronExecutorRuntime() {
return await cronExecutorRuntimeLoader.load();
}
function isCronNestedLaneTaskTimeoutError(err: unknown): boolean {
return isCommandLaneTaskTimeoutError(err, CommandLane.CronNested);
}
@@ -211,7 +207,7 @@ export async function runCronIsolatedAgentTurn(params: {
bindAgentRunTaskRunId(initialSessionId, runContextOwnerToken, taskRunId);
}
}
const { executeCronRun } = await loadCronExecutorRuntime();
const { executeCronRun } = await cronExecutorRuntimeLoader.load();
const executionParams: Parameters<typeof executeCronRun>[0] = {
cfg: params.cfg,
cfgWithAgentDefaults: prepared.context.cfgWithAgentDefaults,
+1 -5
View File
@@ -8,10 +8,6 @@ const gatewayCallRuntimeLoader = createLazyImportLoader(
() => import("../../gateway/call.runtime.js"),
);
async function loadGatewayCallRuntime(): Promise<typeof import("../../gateway/call.runtime.js")> {
return await gatewayCallRuntimeLoader.load();
}
export type CronRunSessionCleanupOutcome =
| "not-requested"
| "deleted"
@@ -33,7 +29,7 @@ export async function cleanupCronRunSessionAfterRun(params: {
}
params.beforeDelete?.();
try {
const { callGateway } = await loadGatewayCallRuntime();
const { callGateway } = await gatewayCallRuntimeLoader.load();
const result = await callGateway<{ deleted?: boolean }>({
method: "sessions.delete",
params: {
+1 -3
View File
@@ -178,7 +178,6 @@ export function resolveCronSession(params: {
let isNewSession: boolean;
let systemSent: boolean;
let resetBoundaryPending: { reason: "cron-stale"; sessionFile: string } | undefined;
let staleBoundaryReset = false;
if (!params.forceNew && entry?.sessionId) {
// Cron/webhook sessions follow the direct reset policy so scheduled turns
@@ -211,7 +210,6 @@ export function resolveCronSession(params: {
isNewSession = true;
systemSent = false;
if (!sourceSessionDiffers) {
staleBoundaryReset = true;
resetBoundaryPending = { reason: "cron-stale", sessionFile: params.sessionKey };
}
}
@@ -222,7 +220,7 @@ export function resolveCronSession(params: {
}
const previousSessionId =
isNewSession && !sourceSessionDiffers && !staleBoundaryReset ? entry?.sessionId : undefined;
isNewSession && !sourceSessionDiffers && !resetBoundaryPending ? entry?.sessionId : undefined;
clearBootstrapSnapshotOnSessionRollover({
sessionKey: params.sessionKey,
previousSessionId,
+13 -11
View File
@@ -1,3 +1,4 @@
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
import { parseDurationMs } from "../cli/parse-duration.js";
import type { CronPacing } from "./types.js";
@@ -8,16 +9,15 @@ type CronPacingBounds = {
};
function parsePositivePacingDuration(value: string, field: "min" | "max"): number {
let durationMs: number;
try {
durationMs = parseDurationMs(value);
const durationMs = parseDurationMs(value);
if (durationMs > 0) {
return durationMs;
}
} catch {
throw new Error(`cron pacing ${field} must be a positive duration`);
// Normalize parser details into the cron configuration contract below.
}
if (durationMs <= 0) {
throw new Error(`cron pacing ${field} must be a positive duration`);
}
return durationMs;
throw new Error(`cron pacing ${field} must be a positive duration`);
}
/** Validates pacing strings and returns their millisecond bounds. */
@@ -40,11 +40,13 @@ export function resolvePacedNextRunAtMs(params: {
nowMs: number;
delayMs: number;
pacing: CronPacing;
}): number {
}): number | undefined {
const { minMs, maxMs } = parseCronPacingBounds(params.pacing);
const proposedAtMs = params.nowMs + params.delayMs;
return Math.min(
params.nowMs + (maxMs ?? Number.POSITIVE_INFINITY),
Math.max(params.nowMs + (minMs ?? 0), proposedAtMs),
return asDateTimestampMs(
Math.min(
params.nowMs + (maxMs ?? Number.POSITIVE_INFINITY),
Math.max(params.nowMs + (minMs ?? 0), proposedAtMs),
),
);
}
+2
View File
@@ -220,6 +220,8 @@ describe("parseAbsoluteTimeMs", () => {
// JavaScript Date range ends at +100,000,000 days
const maxValid = new Date(8640000000000000).getTime();
expect(parseAbsoluteTimeMs(maxValid.toString())).toBe(maxValid);
expect(parseAbsoluteTimeMs(new Date(maxValid).toISOString())).toBe(maxValid);
expect(parseAbsoluteTimeMs("+275760-09-13T00:00:00.001Z")).toBeNull();
});
});
+2 -2
View File
@@ -3,8 +3,8 @@ import { parseStrictPositiveInteger } from "../infra/parse-finite-number.js";
import { hasValidIsoCalendarComponents } from "../shared/iso-time.js";
const ISO_TZ_RE = /(Z|[+-]\d{2}:?\d{2})$/i;
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
const ISO_DATE_TIME_RE = /^\d{4}-\d{2}-\d{2}[Tt]/;
const ISO_DATE_RE = /^(?:[+-]\d{6}|\d{4})-\d{2}-\d{2}$/;
const ISO_DATE_TIME_RE = /^(?:[+-]\d{6}|\d{4})-\d{2}-\d{2}[Tt]/;
function normalizeUtcIso(raw: string) {
if (ISO_TZ_RE.test(raw)) {
+81 -21
View File
@@ -1,12 +1,62 @@
import { compileSafeRegex } from "../security/safe-regex.js";
/** Validates persisted cron job records before loading them from disk/state. */
import {
asSafeIntegerInRange,
MAX_DATE_TIMESTAMP_MS,
} from "@openclaw/normalization-core/number-coercion";
import { asRecord } from "@openclaw/normalization-core/record-coerce";
import { compileSafeRegex } from "../security/safe-regex.js";
import { parseAbsoluteTimeMs } from "./parse.js";
import type { CronJobState } from "./types.js";
const CRON_STATE_TIMESTAMP_FIELDS = [
"nextRunAtMs",
"scheduleActivatedAtMs",
"startupCatchupAtMs",
"pacedNextRunAtMs",
"forcePreservedNextRunAtMs",
"queuedAtMs",
"runningAtMs",
"lastRunAtMs",
"lastFailureAlertAtMs",
"lastTriggerEvalAtMs",
"lastTriggerFireAtMs",
"streamLastStartedAtMs",
"streamLastExitAtMs",
] as const satisfies readonly (keyof CronJobState)[];
function isValidStateTimestamp(value: unknown): boolean {
return asSafeIntegerInRange(value, { min: 0, max: MAX_DATE_TIMESTAMP_MS }) !== undefined;
}
function getInvalidCronJobStateTimestampField(state: unknown): string | undefined {
const record = asRecord(state);
const field = CRON_STATE_TIMESTAMP_FIELDS.find(
(key) => record[key] !== undefined && !isValidStateTimestamp(record[key]),
);
if (field) {
return field;
}
const atMs = asRecord(record.autoDisabled).atMs;
return atMs !== undefined && !isValidStateTimestamp(atMs) ? "autoDisabled.atMs" : undefined;
}
/** Rejects caller-authored state timestamps that cannot round-trip through Date and SQLite. */
export function assertCronJobStateTimestamps(state: Partial<CronJobState>): void {
const invalidField = getInvalidCronJobStateTimestampField(state);
if (invalidField) {
throw new Error(
`cron state.${invalidField} must be a non-negative Date-valid integer timestamp`,
);
}
}
/** Structural rejection code for persisted cron jobs that cannot be loaded safely. */
type InvalidPersistedCronJobReason =
| "missing-id"
| "missing-schedule"
| "invalid-schedule"
| "invalid-state"
| "unsatisfiable-schedule"
| "invalid-trigger"
| "missing-payload"
| "invalid-payload";
@@ -19,6 +69,9 @@ export function getInvalidPersistedCronJobReason(
if (typeof id !== "string" || !id.trim()) {
return "missing-id";
}
if (getInvalidCronJobStateTimestampField(candidate.state)) {
return "invalid-state";
}
const schedule = candidate.schedule;
if (!schedule || Array.isArray(schedule)) {
return "missing-schedule";
@@ -50,13 +103,24 @@ export function getInvalidPersistedCronJobReason(
}
if (scheduleKind === "every") {
const everyMs = scheduleRecord.everyMs;
if (typeof everyMs !== "number" || !Number.isFinite(everyMs) || everyMs <= 0) {
const anchorMs = scheduleRecord.anchorMs;
if (
asSafeIntegerInRange(everyMs, { min: 1, max: MAX_DATE_TIMESTAMP_MS }) === undefined ||
(anchorMs !== undefined &&
asSafeIntegerInRange(anchorMs, { min: 0, max: MAX_DATE_TIMESTAMP_MS }) === undefined)
) {
return "invalid-schedule";
}
}
if (scheduleKind === "cron") {
const expr = scheduleRecord.expr;
if (typeof expr !== "string" || expr.trim().length === 0) {
const staggerMs = scheduleRecord.staggerMs;
if (
typeof expr !== "string" ||
expr.trim().length === 0 ||
(staggerMs !== undefined &&
asSafeIntegerInRange(staggerMs, { min: 0, max: MAX_DATE_TIMESTAMP_MS }) === undefined)
) {
return "invalid-schedule";
}
}
@@ -74,7 +138,7 @@ export function getInvalidPersistedCronJobReason(
// one such throw would abort the single-pass stream reconcile and block
// every valid stream job. Quarantine the row here instead.
const batchFieldValid = (value: unknown) =>
value === undefined || (typeof value === "number" && Number.isSafeInteger(value));
value === undefined || asSafeIntegerInRange(value, {}) !== undefined;
if (
!Array.isArray(command) ||
command.length === 0 ||
@@ -123,17 +187,19 @@ export function getInvalidPersistedCronJobReason(
) {
return "invalid-payload";
}
if (payloadKind === "systemEvent") {
const text = payloadRecord.text;
if (typeof text !== "string") {
return "invalid-payload";
}
}
if (payloadKind === "agentTurn") {
const message = payloadRecord.message;
if (typeof message !== "string" || message.trim().length === 0) {
return "invalid-payload";
}
const requiredText =
payloadKind === "systemEvent"
? payloadRecord.text
: payloadKind === "agentTurn"
? payloadRecord.message
: payloadKind === "script"
? payloadRecord.script
: undefined;
if (
(payloadKind === "systemEvent" || payloadKind === "agentTurn" || payloadKind === "script") &&
(typeof requiredText !== "string" || (payloadKind !== "systemEvent" && !requiredText.trim()))
) {
return "invalid-payload";
}
if (payloadKind === "command") {
const argv = payloadRecord.argv;
@@ -148,11 +214,5 @@ export function getInvalidPersistedCronJobReason(
return "invalid-payload";
}
}
if (payloadKind === "script") {
const script = payloadRecord.script;
if (typeof script !== "string" || script.trim().length === 0) {
return "invalid-payload";
}
}
return null;
}
+7 -1
View File
@@ -4,10 +4,12 @@ import { toPublicCronJob } from "./public-job.js";
import type { CronStoredJob } from "./types.js";
describe("toPublicCronJob", () => {
it("strips scheduler-only pacing slots without mutating stored state", () => {
it("strips scheduler-only state without mutating the stored job", () => {
const job = makeCronJob({
state: {
nextRunAtMs: 2_000,
queuedAtMs: 1_900,
startupCatchupAtMs: 2_000,
pacedNextRunAtMs: 2_000,
forcePreservedNextRunAtMs: 2_000,
},
@@ -15,8 +17,12 @@ describe("toPublicCronJob", () => {
const publicJob = toPublicCronJob(job);
expect(publicJob.state.queuedAtMs).toBeUndefined();
expect(publicJob.state.startupCatchupAtMs).toBeUndefined();
expect(publicJob.state.pacedNextRunAtMs).toBeUndefined();
expect(publicJob.state.forcePreservedNextRunAtMs).toBeUndefined();
expect(job.state.queuedAtMs).toBe(1_900);
expect(job.state.startupCatchupAtMs).toBe(2_000);
expect(job.state.pacedNextRunAtMs).toBe(2_000);
expect(job.state.forcePreservedNextRunAtMs).toBe(2_000);
});
+19 -21
View File
@@ -1,4 +1,8 @@
/** Builds stable identities for cron scheduling inputs. */
import {
asSafeIntegerInRange,
parseStrictFiniteNumber,
} from "@openclaw/normalization-core/number-coercion";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { parseCronPacingBounds } from "./pacing.js";
import { coerceFiniteScheduleNumber } from "./schedule-number.js";
@@ -13,12 +17,16 @@ function readString(record: Record<string, unknown>, key: string): string | unde
return normalizeOptionalString(record[key]);
}
function readNumber(record: Record<string, unknown>, key: string): number | undefined {
function readScheduleTime(record: Record<string, unknown>, key: string): number | undefined {
return coerceFiniteScheduleNumber(record[key]);
}
function readStaggerMs(record: Record<string, unknown>): number | undefined {
return normalizeCronStaggerMs(record.staggerMs);
function readNumber(record: Record<string, unknown>, key: string): number | undefined {
const parsed = parseStrictFiniteNumber(record[key]);
return asSafeIntegerInRange(parsed, {
min: Number.MIN_SAFE_INTEGER,
max: Number.MAX_SAFE_INTEGER,
});
}
function schedulePayloadFromRecord(schedule: Record<string, unknown>):
@@ -39,10 +47,10 @@ function schedulePayloadFromRecord(schedule: Record<string, unknown>):
const rawKind = readString(schedule, "kind")?.toLowerCase();
const expr = readString(schedule, "expr");
const at = readString(schedule, "at");
const everyMs = readNumber(schedule, "everyMs");
const anchorMs = readNumber(schedule, "anchorMs");
const everyMs = readScheduleTime(schedule, "everyMs");
const anchorMs = readScheduleTime(schedule, "anchorMs");
const tz = readString(schedule, "tz");
const staggerMs = readStaggerMs(schedule);
const staggerMs = normalizeCronStaggerMs(schedule.staggerMs);
const kind =
// Infer legacy shorthand schedule shapes when kind is missing so timer
// identity remains stable across old persisted jobs and normalized jobs.
@@ -96,15 +104,6 @@ function schedulePayloadFromRecord(schedule: Record<string, unknown>):
return undefined;
}
function resolveSchedulePayload(
job: CronScheduleIdentityInput,
): ReturnType<typeof schedulePayloadFromRecord> {
if (job.schedule && typeof job.schedule === "object" && !Array.isArray(job.schedule)) {
return schedulePayloadFromRecord(job.schedule as Record<string, unknown>);
}
return undefined;
}
function resolvePacingPayload(
job: CronScheduleIdentityInput,
): { minMs?: number; maxMs?: number } | null | undefined {
@@ -126,7 +125,10 @@ function resolvePacingPayload(
/** Builds a stable scheduling identity for deciding whether stored timer state is still valid. */
export function tryCronScheduleIdentity(job: CronScheduleIdentityInput): string | undefined {
const schedule = resolveSchedulePayload(job);
const schedule =
job.schedule && typeof job.schedule === "object" && !Array.isArray(job.schedule)
? schedulePayloadFromRecord(job.schedule as Record<string, unknown>)
: undefined;
const pacing = resolvePacingPayload(job);
if (!schedule || pacing === null) {
return undefined;
@@ -147,9 +149,5 @@ export function cronSchedulingInputsEqual(
): boolean {
const previousIdentity = tryCronScheduleIdentity(previous);
const nextIdentity = tryCronScheduleIdentity(next);
return (
previousIdentity !== undefined &&
nextIdentity !== undefined &&
previousIdentity === nextIdentity
);
return previousIdentity !== undefined && previousIdentity === nextIdentity;
}
+7 -4
View File
@@ -1,8 +1,11 @@
/** Coerces cron schedule number fields with strict safe-range parsing. */
import { parseStrictFiniteNumber } from "@openclaw/normalization-core/number-coercion";
/** Coerces cron schedule time fields with strict Date-range parsing. */
import {
asDateTimestampMs,
parseStrictFiniteNumber,
} from "@openclaw/normalization-core/number-coercion";
/** Coerces schedule numeric fields without accepting partial, non-finite, or unsafe values. */
/** Coerces temporal schedule fields without accepting partial, non-finite, or invalid-Date values. */
export function coerceFiniteScheduleNumber(value: unknown): number | undefined {
const parsed = parseStrictFiniteNumber(value);
return parsed !== undefined && Math.abs(parsed) <= Number.MAX_SAFE_INTEGER ? parsed : undefined;
return asDateTimestampMs(parsed);
}
+36 -5
View File
@@ -1,11 +1,9 @@
// Cron schedule tests cover schedule parsing and next-run calculations.
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { Cron } from "croner";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
coerceFiniteScheduleNumber,
computeNextRunAtMs,
computePreviousRunAtMs,
} from "./schedule.js";
import { coerceFiniteScheduleNumber } from "./schedule-number.js";
import { computeNextRunAtMs, computePreviousRunAtMs } from "./schedule.js";
import {
clearCronScheduleCacheForTest,
getCronScheduleCacheMaxForTest,
@@ -385,6 +383,38 @@ describe("cron schedule", () => {
expect(next).toBe(anchor + 60_000);
});
it("rejects every schedule numbers outside the ECMAScript Date range", () => {
expect(
computeNextRunAtMs({ kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS + 1 }, 0),
).toBeUndefined();
expect(
computeNextRunAtMs({ kind: "every", everyMs: 1, anchorMs: MAX_DATE_TIMESTAMP_MS + 1 }, 0),
).toBeUndefined();
expect(computeNextRunAtMs({ kind: "every", everyMs: 0.5, anchorMs: 0 }, 0)).toBeUndefined();
expect(computeNextRunAtMs({ kind: "every", everyMs: 1, anchorMs: -1 }, 0)).toBeUndefined();
});
it("does not return an every occurrence outside the ECMAScript Date range", () => {
const anchorMs = MAX_DATE_TIMESTAMP_MS - 1;
expect(computeNextRunAtMs({ kind: "every", everyMs: 2, anchorMs }, anchorMs)).toBeUndefined();
});
it.each([
["NaN", Number.NaN],
["positive infinity", Number.POSITIVE_INFINITY],
["negative infinity", Number.NEGATIVE_INFINITY],
["above Date range", MAX_DATE_TIMESTAMP_MS + 1],
["below Date range", -MAX_DATE_TIMESTAMP_MS - 1],
])("returns undefined instead of throwing for an invalid %s cursor", (_label, nowMs) => {
expect(computeNextRunAtMs({ kind: "every", everyMs: 60_000 }, nowMs)).toBeUndefined();
expect(
computeNextRunAtMs({ kind: "cron", expr: "0 * * * *", tz: "UTC" }, nowMs),
).toBeUndefined();
expect(
computePreviousRunAtMs({ kind: "cron", expr: "0 * * * *", tz: "UTC" }, nowMs),
).toBeUndefined();
});
it("never returns a past timestamp for Asia/Shanghai daily schedule (#30351)", () => {
const nowMs = Date.parse("2026-03-01T00:00:00.000Z");
const next = computeNextRunAtMs(
@@ -527,6 +557,7 @@ describe("coerceFiniteScheduleNumber", () => {
expect(coerceFiniteScheduleNumber("0x10")).toBeUndefined();
expect(coerceFiniteScheduleNumber(Number.NaN)).toBeUndefined();
expect(coerceFiniteScheduleNumber(Infinity)).toBeUndefined();
expect(coerceFiniteScheduleNumber(MAX_DATE_TIMESTAMP_MS + 1)).toBeUndefined();
expect(coerceFiniteScheduleNumber(Number.MAX_SAFE_INTEGER + 1)).toBeUndefined();
expect(coerceFiniteScheduleNumber(String(Number.MAX_SAFE_INTEGER + 1))).toBeUndefined();
expect(coerceFiniteScheduleNumber(null)).toBeUndefined();
+13 -5
View File
@@ -1,4 +1,5 @@
/** Computes at/every/cron schedule timestamps with bounded Croner caching. */
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { Cron, CronDate } from "croner";
import { parseOffsetlessIsoDateTimeInTimeZone } from "../infra/format-time/parse-offsetless-zoned-datetime.js";
@@ -7,8 +8,6 @@ import { parseAbsoluteTimeMs } from "./parse.js";
import { coerceFiniteScheduleNumber } from "./schedule-number.js";
import type { CronSchedule } from "./types.js";
export { coerceFiniteScheduleNumber } from "./schedule-number.js";
const CRON_EVAL_CACHE_MAX = 512;
const DAY_MS = 86_400_000;
const cronEvalCache = new Map<string, Cron>();
@@ -211,6 +210,9 @@ function resolveValidatedNextCronOccurrenceMs(
/** Computes the next scheduled run timestamp after now for at/every/cron schedules. */
export function computeNextRunAtMs(schedule: CronSchedule, nowMs: number): number | undefined {
if (asDateTimestampMs(nowMs) === undefined) {
return undefined;
}
if (schedule.kind === "at") {
const atMs = parseAbsoluteTimeMs(schedule.at);
if (atMs === null) {
@@ -224,15 +226,21 @@ export function computeNextRunAtMs(schedule: CronSchedule, nowMs: number): numbe
if (everyMsRaw === undefined) {
return undefined;
}
const everyMs = Math.max(1, Math.floor(everyMsRaw));
const everyMs = Math.floor(everyMsRaw);
if (everyMs < 1) {
return undefined;
}
const anchorRaw = coerceFiniteScheduleNumber(schedule.anchorMs);
if (schedule.anchorMs !== undefined && (anchorRaw === undefined || anchorRaw < 0)) {
return undefined;
}
const anchor = Math.max(0, Math.floor(anchorRaw ?? nowMs));
if (nowMs < anchor) {
return anchor;
}
const elapsed = nowMs - anchor;
const steps = Math.floor(elapsed / everyMs) + 1;
return anchor + steps * everyMs;
return asDateTimestampMs(anchor + steps * everyMs);
}
if (schedule.kind === "on-exit" || schedule.kind === "stream") {
@@ -281,7 +289,7 @@ export function computeNextRunAtMs(schedule: CronSchedule, nowMs: number): numbe
/** Computes the previous cron-expression run timestamp before now. */
export function computePreviousRunAtMs(schedule: CronSchedule, nowMs: number): number | undefined {
if (schedule.kind !== "cron") {
if (schedule.kind !== "cron" || asDateTimestampMs(nowMs) === undefined) {
return undefined;
}
const cron = resolveCronFromSchedule(schedule);
+1 -1
View File
@@ -1,6 +1,6 @@
// Cron reschedule regression tests cover @schedule jobs after schedule changes.
import { describe, expect, it } from "vitest";
import { computeJobNextRunAtMs } from "./service/jobs.js";
import { computeJobNextRunAtMs } from "./service/jobs-scheduling.js";
import type { CronJob } from "./types.js";
const ORIGINAL_AT_MS = Date.parse("2026-02-22T10:00:00.000Z");
+1 -1
View File
@@ -1,7 +1,7 @@
// Daily skip regression tests cover missed-run handling for daily cron jobs.
import { describe, expect, it } from "vitest";
import { createMockCronStateForJobs } from "./service.test-harness.js";
import { recomputeNextRuns, recomputeNextRunsForMaintenance } from "./service/jobs.js";
import { recomputeNextRuns, recomputeNextRunsForMaintenance } from "./service/jobs-scheduling.js";
import type { CronJob } from "./types.js";
/**
+40 -1
View File
@@ -1,6 +1,11 @@
// Every-next-run regression tests cover next-run calculations for repeating jobs.
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it } from "vitest";
import { computeJobNextRunAtMs } from "./service/jobs.js";
import {
computeJobNextRunAtMs,
hasScheduledNextRunAtMs,
resolveJobErrorBackoffUntilMs,
} from "./service/jobs-scheduling.js";
import type { CronJob } from "./types.js";
const EVERY_30_MIN_MS = 30 * 60_000;
@@ -57,4 +62,38 @@ describe("Cron issue #22895 interval scheduling", () => {
const next = computeJobNextRunAtMs(job, nowMs);
expect(next).toBe(Date.parse("2026-02-22T10:44:00.000Z"));
});
it("does not return an invalid Date when last-run cadence exceeds the timestamp range", () => {
const job = createEveryJob({ lastRunAtMs: MAX_DATE_TIMESTAMP_MS });
job.schedule = { kind: "every", everyMs: 1, anchorMs: 0 };
expect(computeJobNextRunAtMs(job, MAX_DATE_TIMESTAMP_MS - 1)).toBeUndefined();
expect(hasScheduledNextRunAtMs(MAX_DATE_TIMESTAMP_MS + 1)).toBe(false);
});
it("preserves the inclusive maximum Date timestamp", () => {
const job = createEveryJob({ lastRunAtMs: MAX_DATE_TIMESTAMP_MS - 1 });
job.schedule = { kind: "every", everyMs: 1, anchorMs: 0 };
expect(computeJobNextRunAtMs(job, MAX_DATE_TIMESTAMP_MS - 2)).toBe(MAX_DATE_TIMESTAMP_MS);
expect(hasScheduledNextRunAtMs(MAX_DATE_TIMESTAMP_MS)).toBe(true);
});
it("rejects malformed intervals, anchors, and overflowing error backoff", () => {
const job = createEveryJob({
lastRunAtMs: MAX_DATE_TIMESTAMP_MS,
lastDurationMs: 1,
lastStatus: "error",
consecutiveErrors: 1,
});
expect(resolveJobErrorBackoffUntilMs(job)).toBeUndefined();
job.state.lastRunAtMs = MAX_DATE_TIMESTAMP_MS - 30_000;
job.state.lastDurationMs = 0;
expect(resolveJobErrorBackoffUntilMs(job)).toBe(MAX_DATE_TIMESTAMP_MS);
job.schedule = { kind: "every", everyMs: 0.5, anchorMs: 0 };
expect(computeJobNextRunAtMs(job, ANCHOR_MS)).toBeUndefined();
job.schedule = { kind: "every", everyMs: 1, anchorMs: -1 };
expect(computeJobNextRunAtMs(job, ANCHOR_MS)).toBeUndefined();
});
});
+107 -4
View File
@@ -1,15 +1,14 @@
// Cron service job tests cover job creation, updates, and runtime scheduling.
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it } from "vitest";
import {
applyDeclarativeJobSpec,
applyJobPatch,
computeJobNextRunAtMs,
computeJobPreviousRunAtOrBeforeMs,
createJob,
nextWakeAtMs,
recomputeNextRuns,
recomputeNextRunsForMaintenance,
} from "./service/jobs.js";
} from "./service/jobs-scheduling.js";
import { applyDeclarativeJobSpec, applyJobPatch, createJob } from "./service/jobs.js";
import type { CronServiceState } from "./service/state.js";
import type { CronJob, CronJobPatch } from "./types.js";
@@ -558,6 +557,49 @@ function createMockState(
} as unknown as CronServiceState;
}
describe("time schedule validation", () => {
const now = Date.parse("2026-08-10T00:00:00.000Z");
const input = (anchorMs?: number) => ({
name: "Date boundary interval",
enabled: true,
schedule: { kind: "every" as const, everyMs: MAX_DATE_TIMESTAMP_MS, anchorMs },
sessionTarget: "main" as const,
wakeMode: "now" as const,
payload: { kind: "systemEvent" as const, text: "tick" },
});
it("rejects intervals with no representable next run while preserving the inclusive boundary", () => {
expect(() => createJob(createMockState(now), input())).toThrow(
"cron every schedule has no upcoming run time and would never fire",
);
expect(createJob(createMockState(now), input(0)).state.nextRunAtMs).toBe(MAX_DATE_TIMESTAMP_MS);
});
it("rejects invalid one-shot timestamps at the service boundary", () => {
const maxAt = new Date(MAX_DATE_TIMESTAMP_MS).toISOString();
expect(
createJob(createMockState(now), {
name: "Maximum one-shot",
enabled: true,
schedule: { kind: "at", at: maxAt },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
}).state.nextRunAtMs,
).toBe(MAX_DATE_TIMESTAMP_MS);
expect(() =>
createJob(createMockState(now), {
name: "Invalid one-shot",
enabled: true,
schedule: { kind: "at", at: String(MAX_DATE_TIMESTAMP_MS + 1) },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
}),
).toThrow("Date-valid absolute timestamp");
});
});
describe("announce delivery channel validation", () => {
const now = Date.parse("2026-08-02T12:00:00.000Z");
const configuredChannels = ["reef", "discord"];
@@ -761,6 +803,35 @@ describe("cron tool authority defaults", () => {
});
});
it("repairs a missing anchor when converging an unchanged every schedule", () => {
const createdAtMs = now - 30_000;
const job = createJob(createMockState(now), {
name: "legacy declaration",
enabled: true,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
});
job.createdAtMs = createdAtMs;
job.schedule = { kind: "every", everyMs: 60_000 };
applyDeclarativeJobSpec(
job,
{
name: job.name,
enabled: true,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
},
{ enabledExplicit: false, nowMs: now },
);
expect(job.schedule).toEqual({ kind: "every", everyMs: 60_000, anchorMs: createdAtMs });
});
it("adopts explicit authority when a declaration becomes tool-bearing", () => {
const job: CronJob = {
id: "declared-trigger",
@@ -1242,6 +1313,38 @@ describe("cron stagger defaults", () => {
});
});
it("preserves staggering when declarative convergence keeps the cron expression", () => {
const now = Date.now();
const job = createJob(createMockState(now), {
name: "declared hourly",
enabled: true,
schedule: { kind: "cron", expr: "0 * * * *", tz: "UTC", staggerMs: 120_000 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
});
applyDeclarativeJobSpec(
job,
{
name: job.name,
enabled: true,
schedule: { kind: "cron", expr: "0 * * * *", tz: "America/Los_Angeles" },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
},
{ enabledExplicit: false, nowMs: now },
);
expect(job.schedule).toEqual({
kind: "cron",
expr: "0 * * * *",
tz: "America/Los_Angeles",
staggerMs: 120_000,
});
});
it("applies default stagger when switching from every to top-of-hour cron", () => {
const now = Date.now();
const job: CronJob = {
@@ -1,7 +1,7 @@
// Top-of-hour stagger tests cover spreading jobs that would otherwise collide.
import crypto from "node:crypto";
import { describe, expect, it, vi } from "vitest";
import { computeJobNextRunAtMs } from "./service/jobs.js";
import { computeJobNextRunAtMs } from "./service/jobs-scheduling.js";
import type { CronJob } from "./types.js";
const DEFAULT_TOP_OF_HOUR_STAGGER_MS = 5 * 60 * 1000;
+1 -1
View File
@@ -1,7 +1,7 @@
// Cron service regression tests cover historical scheduling edge cases.
import { describe, expect, it } from "vitest";
import { createMockCronStateForJobs } from "./service.test-harness.js";
import { recomputeNextRunsForMaintenance } from "./service/jobs.js";
import { recomputeNextRunsForMaintenance } from "./service/jobs-scheduling.js";
import { reserveQueuedCronRun } from "./service/run-admission.js";
import type { CronJob } from "./types.js";
@@ -1,10 +1,13 @@
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it, vi } from "vitest";
import { setupCronServiceSuite } from "./service.test-harness.js";
import { start } from "./service/ops-lifecycle.js";
import { status } from "./service/ops-read.js";
import { createCronServiceState } from "./service/state.js";
import { runMissedJobs } from "./service/timer.js";
import { onTimer } from "./service/timer.test-support.js";
import { saveCronStore } from "./store.js";
import * as cronStoreModule from "./store.js";
import { loadCronStore, saveCronStore } from "./store.js";
import type { CronJob } from "./types.js";
const { logger: noopLogger, makeStorePath } = setupCronServiceSuite({
@@ -13,6 +16,21 @@ const { logger: noopLogger, makeStorePath } = setupCronServiceSuite({
});
describe("CronService startup catch-up repair scoping", () => {
function createDateBoundaryEveryJob(id: string, nextRunAtMs: number): CronJob {
return {
id,
name: `job-${id}`,
enabled: true,
createdAtMs: nextRunAtMs - 60_000,
updatedAtMs: nextRunAtMs - 60_000,
schedule: { kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS, anchorMs: 0 },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: `tick-${id}` },
state: { nextRunAtMs },
};
}
function createHourlyCronJob(id: string, nextRunAtMs: number): CronJob {
return {
id,
@@ -143,4 +161,92 @@ describe("CronService startup catch-up repair scoping", () => {
state.stopped = true;
await store.cleanup();
});
it("disables startup catch-up deferrals that exceed the Date range", async () => {
const store = await makeStorePath();
const now = MAX_DATE_TIMESTAMP_MS - 2_000;
await saveCronStore(store.storePath, {
version: 1,
jobs: [
createDateBoundaryEveryJob("date-limit-0", now - 60_000),
createDateBoundaryEveryJob("date-limit-1", now - 50_000),
createDateBoundaryEveryJob("date-limit-2", now - 40_000),
],
});
const order: string[] = [];
const deferredAutoDisableReasons = new Set([
"cron:date-limit-1:auto-disabled",
"cron:date-limit-2:auto-disabled",
]);
const enqueueSystemEvent = vi.fn((_text: string, context?: { contextKey?: string }) => {
if (context?.contextKey && deferredAutoDisableReasons.has(context.contextKey)) {
order.push("notify");
}
});
const requestHeartbeat = vi.fn((request: { reason?: string }) => {
if (request.reason && deferredAutoDisableReasons.has(request.reason)) {
order.push("heartbeat");
}
});
const state = createCronServiceState({
cronEnabled: true,
storePath: store.storePath,
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
maxMissedJobsPerRestart: 1,
missedJobStaggerMs: 5_000,
});
const save = cronStoreModule.saveCronJobsStore;
const saveSpy = vi
.spyOn(cronStoreModule, "saveCronJobsStore")
.mockImplementation(async (...args) => {
if (
args[1].jobs.some(
(job) => job.id !== "date-limit-0" && job.state.autoDisabled !== undefined,
)
) {
expect(order).toEqual([]);
const result = await save(...args);
expect(order).toEqual([]);
order.push("persist");
return result;
}
return await save(...args);
});
try {
await runMissedJobs(state);
const deferred = (state.store?.jobs ?? []).filter((job) => job.id !== "date-limit-0");
expect(deferred).toHaveLength(2);
for (const job of deferred) {
expect(job.enabled).toBe(false);
expect(job.state.nextRunAtMs).toBeUndefined();
expect(job.state.startupCatchupAtMs).toBeUndefined();
expect(job.state.autoDisabled).toEqual({
reason: "schedule-errors",
atMs: now,
consecutiveErrors: 1,
});
}
expect(order).toEqual(["persist", "notify", "heartbeat", "notify", "heartbeat"]);
expect((await loadCronStore(store.storePath)).jobs).toEqual(
expect.arrayContaining(
deferred.map((job) =>
expect.objectContaining({
id: job.id,
enabled: false,
state: expect.objectContaining({ autoDisabled: job.state.autoDisabled }),
}),
),
),
);
} finally {
saveSpy.mockRestore();
await store.cleanup();
}
});
});
@@ -1,3 +1,4 @@
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it, vi } from "vitest";
import { CronService } from "./service.js";
import { setupCronServiceSuite } from "./service.test-harness.js";
@@ -177,6 +178,22 @@ describe("cron stream schedule validation", () => {
}
});
it("rejects invalid scheduler timestamps from external event sources", async () => {
const cron = await createCron(true);
try {
const created = await cron.add(streamJob());
await expect(
cron.recordExternalFailure(created.id, "invalid source state", {
startupCatchupAtMs: MAX_DATE_TIMESTAMP_MS + 1,
}),
).rejects.toThrow("cron state.startupCatchupAtMs");
expect(cron.getJob(created.id)?.state.startupCatchupAtMs).toBeUndefined();
} finally {
cron.stop();
}
});
it("rotates logical source identity only when source ownership changes", async () => {
const cron = await createCron(true);
try {
+1 -1
View File
@@ -3,7 +3,7 @@ import { createDeferred } from "../../test/helpers/promise.js";
import type { CronEvent } from "./service.js";
import { CronService } from "./service.js";
import { setupCronServiceSuite } from "./service.test-harness.js";
import { computeJobNextRunAtMs } from "./service/jobs.js";
import { computeJobNextRunAtMs } from "./service/jobs-scheduling.js";
import type { CronServiceDeps } from "./service/state.js";
import { loadCronStore } from "./store.js";
import { cronStoreKey } from "./store/key.js";
+33 -23
View File
@@ -1,14 +1,12 @@
/** Scheduling state and next-run computation for cron jobs. */
import crypto from "node:crypto";
import { expectDefined } from "@openclaw/normalization-core";
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
import { pruneMapToMaxSize } from "../../infra/map-size.js";
import { isCronJobActive } from "../active-jobs.js";
import { parseAbsoluteTimeMs } from "../parse.js";
import {
coerceFiniteScheduleNumber,
computeNextRunAtMs,
computePreviousRunAtMs,
} from "../schedule.js";
import { coerceFiniteScheduleNumber } from "../schedule-number.js";
import { computeNextRunAtMs, computePreviousRunAtMs } from "../schedule.js";
import { resolveCronStaggerMs } from "../stagger.js";
import { createCronStreamSourceIdentity, resolveCronStreamBatching } from "../stream-schedule.js";
import type { CronJob, CronSchedule } from "../types.js";
@@ -56,7 +54,7 @@ export const DEFAULT_ERROR_BACKOFF_SCHEDULE_MS = [
];
function isFiniteTimestamp(value: unknown): value is number {
return typeof value === "number" && Number.isFinite(value);
return asDateTimestampMs(value) !== undefined;
}
/** Returns whether a stored next-run timestamp is finite and schedulable. */
@@ -99,7 +97,7 @@ export function resolveJobErrorBackoffUntilMs(
? Math.max(0, Math.floor(job.state.lastDurationMs))
: 0;
const lastEndedAtMs = job.state.lastRunAtMs + lastDurationMs;
return lastEndedAtMs + errorBackoffMs(consecutiveErrors, scheduleMs);
return asDateTimestampMs(lastEndedAtMs + errorBackoffMs(consecutiveErrors, scheduleMs));
}
function resolveStableCronOffsetMs(jobId: string, staggerMs: number) {
@@ -139,7 +137,7 @@ function computeStaggeredCronNextRunAtMs(job: CronJob, nowMs: number) {
return undefined;
}
const shifted = baseNext + offsetMs;
if (shifted > nowMs) {
if (isFiniteTimestamp(shifted) && shifted > nowMs) {
return shifted;
}
cursorMs = Math.max(cursorMs + 1, baseNext + 1_000);
@@ -167,7 +165,7 @@ function computeStaggeredCronPreviousRunAtMs(job: CronJob, nowMs: number) {
return undefined;
}
const shifted = basePrevious + offsetMs;
if (shifted <= nowMs) {
if (isFiniteTimestamp(shifted) && shifted <= nowMs) {
return shifted;
}
cursorMs = Math.max(0, basePrevious - 1_000);
@@ -178,7 +176,7 @@ function computeStaggeredCronPreviousRunAtMs(job: CronJob, nowMs: number) {
function computeStaggeredCronPreviousRunAtOrBeforeMs(job: CronJob, nowMs: number) {
const previous = computeStaggeredCronPreviousRunAtMs(job, nowMs);
const probeMs = nowMs + 1_000;
if (!Number.isFinite(probeMs)) {
if (!isFiniteTimestamp(probeMs)) {
return previous;
}
@@ -205,7 +203,6 @@ function isStaggeredCronRunAtMs(job: CronJob, runAtMs: number): boolean {
}
function isPendingErrorBackoffSlot(params: {
state: CronServiceState;
job: CronJob;
nextRunAtMs: number;
nowMs: number;
@@ -215,12 +212,8 @@ function isPendingErrorBackoffSlot(params: {
return backoffUntilMs !== undefined && nowMs < backoffUntilMs && nextRunAtMs <= backoffUntilMs;
}
function shouldRepairFutureCronNextRunAtMs(params: {
state: CronServiceState;
job: CronJob;
nowMs: number;
}): boolean {
const { state, job, nowMs } = params;
function shouldRepairFutureCronNextRunAtMs(params: { job: CronJob; nowMs: number }): boolean {
const { job, nowMs } = params;
const nextRun = job.state.nextRunAtMs;
if (
job.schedule.kind !== "cron" ||
@@ -235,7 +228,7 @@ function shouldRepairFutureCronNextRunAtMs(params: {
// Error retries may intentionally use a non-cron future timestamp while
// backoff is pending. Once the retry window has elapsed, stale future cron
// slots should be eligible for the same repair as ordinary schedule state.
if (isPendingErrorBackoffSlot({ state, job, nextRunAtMs: nextRun, nowMs })) {
if (isPendingErrorBackoffSlot({ job, nextRunAtMs: nextRun, nowMs })) {
return false;
}
let naturalNext: number | undefined;
@@ -290,6 +283,14 @@ export function resolveEveryAnchorMs(params: {
return 0;
}
function hasInvalidExplicitEveryAnchor(schedule: { anchorMs?: number }): boolean {
if (schedule.anchorMs === undefined) {
return false;
}
const coerced = coerceFiniteScheduleNumber(schedule.anchorMs);
return coerced === undefined || coerced < 0;
}
/** Finds an in-memory cron job or throws the public unknown-id error. */
export function findJobOrThrow(state: CronServiceState, id: string) {
const job = state.store?.jobs.find((j) => j.id === id);
@@ -310,14 +311,23 @@ export function computeJobNextRunAtMs(job: CronJob, nowMs: number): number | und
return undefined;
}
if (job.schedule.kind === "every") {
if (hasInvalidExplicitEveryAnchor(job.schedule)) {
return undefined;
}
const everyMsRaw = coerceFiniteScheduleNumber(job.schedule.everyMs);
if (everyMsRaw === undefined) {
return undefined;
}
const everyMs = Math.max(1, Math.floor(everyMsRaw));
const everyMs = Math.floor(everyMsRaw);
if (everyMs < 1) {
return undefined;
}
const lastRunAtMs = job.state.lastRunAtMs;
if (typeof lastRunAtMs === "number" && Number.isFinite(lastRunAtMs)) {
if (isFiniteTimestamp(lastRunAtMs)) {
const nextFromLastRun = Math.floor(lastRunAtMs) + everyMs;
if (!isFiniteTimestamp(nextFromLastRun)) {
return undefined;
}
if (nextFromLastRun > nowMs) {
return nextFromLastRun;
}
@@ -422,7 +432,7 @@ function normalizeJobTickState(params: { state: CronServiceState; job: CronJob;
changed = true;
}
if (job.schedule.kind === "every") {
if (job.schedule.kind === "every" && !hasInvalidExplicitEveryAnchor(job.schedule)) {
const normalizedAnchorMs = resolveEveryAnchorMs({
schedule: job.schedule,
fallbackAnchorMs: isFiniteTimestamp(job.createdAtMs) ? job.createdAtMs : nowMs,
@@ -607,7 +617,7 @@ export function recomputeNextRuns(state: CronServiceState): boolean {
const isDueOrMissing = !hasScheduledNextRunAtMs(nextRun) || now >= nextRun;
return (
!hasForcePreservedNextRun &&
(isDueOrMissing || shouldRepairFutureCronNextRunAtMs({ state, job, nowMs: now })) &&
(isDueOrMissing || shouldRepairFutureCronNextRunAtMs({ job, nowMs: now })) &&
recomputeJobNextRunAtMs({ state, job, nowMs: now })
);
});
@@ -681,7 +691,7 @@ export function recomputeNextRunsForMaintenance(
!hasPendingStartupCatchup &&
!hasPendingPacedNextRun &&
!hasForcePreservedNextRun &&
shouldRepairFutureCronNextRunAtMs({ state, job, nowMs: now })
shouldRepairFutureCronNextRunAtMs({ job, nowMs: now })
) {
changed = recomputeJob(job, now) || changed;
} else if (
+12 -2
View File
@@ -6,6 +6,7 @@ import { normalizeAgentId } from "../../routing/session-key.js";
import { compileSafeRegexDetailed } from "../../security/safe-regex.js";
import { resolveCronDeliveryPlan } from "../delivery-plan.js";
import { parseCronPacingBounds } from "../pacing.js";
import { parseAbsoluteTimeMs } from "../parse.js";
import { assertSafeCronSessionTargetId } from "../session-target.js";
import type { CronDelivery, CronJob, CronJobPatch } from "../types.js";
import { normalizeHttpWebhookUrl } from "../webhook-url.js";
@@ -153,17 +154,26 @@ export function assertStreamScheduleSupport(
}
}
export function assertCronExpressionSatisfiable(
export function assertTimeScheduleSatisfiable(
job: CronJob,
nowMs: number,
computeJobNextRunAtMs: (job: CronJob, nowMs: number) => number | undefined,
) {
if (job.schedule.kind !== "cron") {
if (job.schedule.kind === "at") {
if (parseAbsoluteTimeMs(job.schedule.at) === null) {
throw new Error("cron at schedule must contain a Date-valid absolute timestamp");
}
return;
}
if (job.schedule.kind !== "cron" && job.schedule.kind !== "every") {
return;
}
if (computeJobNextRunAtMs({ ...job, enabled: true }, nowMs) !== undefined) {
return;
}
if (job.schedule.kind === "every") {
throw new Error("cron every schedule has no upcoming run time and would never fire");
}
throw new Error(
`cron expression "${job.schedule.expr}" has no upcoming run time and would never fire`,
);
+14
View File
@@ -1,4 +1,5 @@
// Cron job patch tests cover applying partial updates to scheduled jobs.
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it } from "vitest";
import { resolveCronDeliveryPlan, resolveFailureDestination } from "../delivery-plan.js";
import { projectCronJobThroughStorageCodec } from "../store/row-codec.js";
@@ -125,6 +126,19 @@ describe("schedule activation ownership", () => {
expect(job.state.scheduleActivatedAtMs).toBe(456);
});
it.each(["nextRunAtMs", "startupCatchupAtMs", "pacedNextRunAtMs"] as const)(
"rejects out-of-Date-range caller state for %s",
(field) => {
const job = makeJob({ enabled: false });
const patch = {
state: { [field]: MAX_DATE_TIMESTAMP_MS + 1 },
} as CronJobPatch;
expect(() => applyJobPatch(job, patch)).toThrow(`cron state.${field}`);
expect(job.state[field]).toBeUndefined();
},
);
});
describe("applyJobPatch delivery merge", () => {
@@ -1,7 +1,7 @@
// Schedule error isolation tests cover one bad job not blocking other cron jobs.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { CronJob, CronStoreFile } from "../types.js";
import { recomputeNextRuns } from "./jobs.js";
import { recomputeNextRuns } from "./jobs-scheduling.js";
import type { CronServiceState } from "./state.js";
function createMockState(jobs: CronJob[]): CronServiceState {
+194 -185
View File
@@ -7,6 +7,7 @@ import {
import type { CronConfig } from "../../config/types.cron.js";
import { normalizeOptionalAccountId } from "../../routing/account-id.js";
import { resolveCronDeliveryPlan } from "../delivery-plan.js";
import { assertCronJobStateTimestamps } from "../persisted-shape.js";
import type { CronScheduledToolPolicy } from "../scheduled-tool-policy.js";
import { normalizeCronScriptPayload } from "../script-payload.js";
import { normalizeCronStaggerMs, resolveDefaultCronStaggerMs } from "../stagger.js";
@@ -20,6 +21,7 @@ import type {
CronJobCreate,
CronJobPatch,
CronJobState,
CronSchedule,
CronStoredJob,
CronToolsAllowProvenance,
} from "../types.js";
@@ -36,7 +38,7 @@ import {
} from "./jobs-tool-policy.js";
import {
assertAnnounceDeliveryChannelSupport,
assertCronExpressionSatisfiable,
assertTimeScheduleSatisfiable,
assertDeliverySupport,
assertFailureDestinationSupport,
assertMainSessionAgentId,
@@ -54,26 +56,151 @@ import type { CronServiceState } from "./state.js";
const CRON_DECLARATIVE_LABEL_MAX_LENGTH = 200;
type DeliveryValidationOptions = { configuredChannels?: readonly string[] };
export { assertSupportedJobSpec };
type ScheduleNormalizationContext =
| { kind: "create"; nowMs: number }
| { kind: "patch"; previous: CronSchedule }
| { kind: "declarative"; previous: CronSchedule; nowMs: number; fallbackAnchorMs: number };
export {
DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
hasScheduledNextRunAtMs,
resolveJobLastRunStatus,
errorBackoffMs,
resolveJobErrorBackoffUntilMs,
findJobOrThrow,
isJobEnabled,
computeJobNextRunAtMs,
computeJobPreviousRunAtOrBeforeMs,
recordScheduleComputeError,
recomputeNextRuns,
recomputeNextRunsForMaintenance,
nextWakeAtMs,
hasActiveCronRun,
isJobDue,
resolveJobPayloadTextForMain,
} from "./jobs-scheduling.js";
function normalizeJobSchedule(
schedule: CronSchedule,
context: ScheduleNormalizationContext,
): CronSchedule {
if (schedule.kind === "every") {
if (context.kind === "patch") {
return schedule;
}
if (context.kind === "create") {
return {
...schedule,
anchorMs: resolveEveryAnchorMs({ schedule, fallbackAnchorMs: context.nowMs }),
};
}
if (schedule.anchorMs !== undefined) {
return schedule;
}
const anchorMs =
context.previous.kind === "every" && context.previous.everyMs === schedule.everyMs
? resolveEveryAnchorMs({
schedule: context.previous,
fallbackAnchorMs: context.fallbackAnchorMs,
})
: context.nowMs;
return {
...schedule,
anchorMs,
};
}
if (schedule.kind === "cron") {
const explicitStaggerMs = normalizeCronStaggerMs(schedule.staggerMs);
if (explicitStaggerMs !== undefined) {
return { ...schedule, staggerMs: explicitStaggerMs };
}
if (
context.kind !== "create" &&
context.previous.kind === "cron" &&
context.previous.expr === schedule.expr
) {
return { ...schedule, staggerMs: context.previous.staggerMs };
}
const defaultStaggerMs = resolveDefaultCronStaggerMs(schedule.expr);
if (defaultStaggerMs !== undefined) {
return { ...schedule, staggerMs: defaultStaggerMs };
}
return context.kind === "declarative" ? { ...schedule } : schedule;
}
const input = context.kind === "declarative" ? structuredClone(schedule) : schedule;
return normalizeStreamScheduleBounds(input);
}
function normalizeDeclarativeLabel(
value: unknown,
field: "declarationKey" | "displayName",
nullable = false,
): string | undefined {
const normalized = normalizeOptionalString(value);
if (!(nullable && value == null) && value !== undefined && !normalized) {
throw new Error(`cron ${field} must not be blank`);
}
if (normalized && normalized.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
throw new Error(
`cron ${field} must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
);
}
return normalized;
}
type JobValidationContext =
| { kind: "create"; cronConfig?: CronConfig; defaultAgentId?: string; nowMs: number }
| {
kind: "patch";
patch: CronJobPatch;
defaultAgentId?: string;
nowMs?: number;
cronConfig?: CronConfig;
}
| {
kind: "declarative";
input: CronJobCreate;
defaultAgentId?: string;
nowMs: number;
cronConfig?: CronConfig;
};
function validateFullJob(
job: CronStoredJob,
context: JobValidationContext,
configuredChannels?: readonly string[],
) {
const cronConfig = context.cronConfig;
const triggerTouched =
context.kind === "create"
? job.trigger !== undefined
: context.kind === "patch"
? context.patch.trigger != null
: context.input.trigger !== undefined;
const scriptTouched =
context.kind === "create"
? job.payload.kind === "script"
: context.kind === "patch"
? context.patch.payload?.kind === "script"
: context.input.payload.kind === "script";
const streamTouched =
context.kind !== "patch" ||
context.patch.enabled === true ||
context.patch.schedule?.kind === "stream";
const validateCapabilities = () => {
assertTriggerSupport(job, { cronConfig, requireEnabled: triggerTouched });
assertScriptPayloadSupport(job, {
cronConfig,
requireEnabled: scriptTouched,
...(context.kind === "patch" ? { validateSyntax: context.patch.payload !== undefined } : {}),
});
assertStreamScheduleSupport(job, { cronConfig, requireEnabled: streamTouched });
};
if (context.kind === "declarative") {
validateCapabilities();
}
assertSupportedJobSpec(job);
assertPacingSupport(job);
if (context.kind !== "declarative") {
validateCapabilities();
}
assertMainSessionAgentId(job, context.defaultAgentId);
assertDeliverySupport(job);
assertAnnounceDeliveryChannelSupport(
job,
configuredChannels,
context.kind === "patch" ? context.patch : undefined,
);
assertFailureDestinationSupport(job);
const scheduleTouched =
context.kind !== "patch" ||
context.patch.schedule !== undefined ||
context.patch.enabled === true;
if (context.nowMs !== undefined && scheduleTouched) {
assertTimeScheduleSatisfiable(job, context.nowMs, computeJobNextRunAtMs);
}
}
/** Creates a normalized cron job row from public add input and computes its initial schedule. */
export function createJob(
state: CronServiceState,
@@ -85,27 +212,7 @@ export function createJob(
): CronStoredJob {
const now = state.deps.nowMs();
const id = normalizeOptionalString(input.id) ?? crypto.randomUUID();
const schedule =
input.schedule.kind === "every"
? {
...input.schedule,
anchorMs: resolveEveryAnchorMs({
schedule: input.schedule,
fallbackAnchorMs: now,
}),
}
: input.schedule.kind === "cron"
? (() => {
const explicitStaggerMs = normalizeCronStaggerMs(input.schedule.staggerMs);
if (explicitStaggerMs !== undefined) {
return { ...input.schedule, staggerMs: explicitStaggerMs };
}
const defaultStaggerMs = resolveDefaultCronStaggerMs(input.schedule.expr);
return defaultStaggerMs !== undefined
? { ...input.schedule, staggerMs: defaultStaggerMs }
: input.schedule;
})()
: normalizeStreamScheduleBounds(input.schedule);
const schedule = normalizeJobSchedule(input.schedule, { kind: "create", nowMs: now });
const deleteAfterRun =
typeof input.deleteAfterRun === "boolean"
? input.deleteAfterRun
@@ -113,24 +220,8 @@ export function createJob(
? true
: undefined;
const enabled = typeof input.enabled === "boolean" ? input.enabled : true;
const declarationKey = normalizeOptionalString(input.declarationKey);
if (input.declarationKey !== undefined && !declarationKey) {
throw new Error("cron declarationKey must not be blank");
}
if (declarationKey && declarationKey.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
throw new Error(
`cron declarationKey must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
);
}
const displayName = normalizeOptionalString(input.displayName);
if (input.displayName !== undefined && !displayName) {
throw new Error("cron displayName must not be blank");
}
if (displayName && displayName.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
throw new Error(
`cron displayName must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
);
}
const declarationKey = normalizeDeclarativeLabel(input.declarationKey, "declarationKey");
const displayName = normalizeDeclarativeLabel(input.displayName, "displayName");
const ownerAgentId = normalizeOptionalAgentId(input.owner?.agentId);
const ownerSessionKey = normalizeOptionalString(input.owner?.sessionKey);
const ownerAccountId = normalizeOptionalAccountId(input.owner?.accountId);
@@ -138,6 +229,8 @@ export function createJob(
// Schedule activation is stamped only by committed scheduling mutations.
// Accepting caller state here would let imports spoof restart catch-up ownership.
delete initialState.scheduleActivatedAtMs;
delete initialState.autoDisabled;
assertCronJobStateTimestamps(initialState);
const job: CronStoredJob = {
id,
...(declarationKey ? { declarationKey } : {}),
@@ -186,25 +279,16 @@ export function createJob(
explicitlyMutatesToolsAllow: true,
toolsAllowProvenance: opts?.toolsAllowProvenance,
});
assertSupportedJobSpec(job);
assertPacingSupport(job);
assertTriggerSupport(job, {
cronConfig: state.deps.cronConfig,
requireEnabled: job.trigger !== undefined,
});
assertScriptPayloadSupport(job, {
cronConfig: state.deps.cronConfig,
requireEnabled: job.payload.kind === "script",
});
assertStreamScheduleSupport(job, {
cronConfig: state.deps.cronConfig,
requireEnabled: true,
});
assertMainSessionAgentId(job, state.deps.defaultAgentId);
assertDeliverySupport(job);
assertAnnounceDeliveryChannelSupport(job, opts?.configuredChannels);
assertFailureDestinationSupport(job);
assertCronExpressionSatisfiable(job, now, computeJobNextRunAtMs);
validateFullJob(
job,
{
kind: "create",
cronConfig: state.deps.cronConfig,
defaultAgentId: state.deps.defaultAgentId,
nowMs: now,
},
opts?.configuredChannels,
);
job.state.nextRunAtMs = computeJobNextRunAtMs(job, now);
return job;
}
@@ -231,15 +315,7 @@ export function applyJobPatch(
job.description = normalizeOptionalString(patch.description);
}
if ("displayName" in patch) {
const displayName = normalizeOptionalString(patch.displayName);
if (patch.displayName !== null && patch.displayName !== undefined && !displayName) {
throw new Error("cron displayName must not be blank");
}
if (displayName && displayName.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
throw new Error(
`cron displayName must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
);
}
const displayName = normalizeDeclarativeLabel(patch.displayName, "displayName", true);
if (displayName) {
job.displayName = displayName;
} else {
@@ -266,24 +342,7 @@ export function applyJobPatch(
delete job.deleteAfterRun;
}
if (patch.schedule) {
if (patch.schedule.kind === "cron") {
const explicitStaggerMs = normalizeCronStaggerMs(patch.schedule.staggerMs);
if (explicitStaggerMs !== undefined) {
job.schedule = { ...patch.schedule, staggerMs: explicitStaggerMs };
} else if (job.schedule.kind === "cron" && job.schedule.expr === patch.schedule.expr) {
// Metadata-only resaves keep the existing stagger, but a replacement
// expression owns a fresh default and must not inherit stale timing.
job.schedule = { ...patch.schedule, staggerMs: job.schedule.staggerMs };
} else {
const defaultStaggerMs = resolveDefaultCronStaggerMs(patch.schedule.expr);
job.schedule =
defaultStaggerMs !== undefined
? { ...patch.schedule, staggerMs: defaultStaggerMs }
: patch.schedule;
}
} else {
job.schedule = normalizeStreamScheduleBounds(patch.schedule);
}
job.schedule = normalizeJobSchedule(patch.schedule, { kind: "patch", previous: job.schedule });
}
if ("trigger" in patch) {
if (patch.trigger === null || patch.trigger === undefined) {
@@ -360,6 +419,7 @@ export function applyJobPatch(
// alone owns the boundary that decides whether restart catch-up can run.
delete statePatch.scheduleActivatedAtMs;
delete statePatch.autoDisabled;
assertCronJobStateTimestamps(statePatch);
job.state = { ...job.state, ...statePatch };
}
if (patch.enabled === true) {
@@ -389,34 +449,17 @@ export function applyJobPatch(
job.state.streamLastStartedAtMs = undefined;
job.state.streamLastExitAtMs = undefined;
}
assertSupportedJobSpec(job);
assertPacingSupport(job);
assertTriggerSupport(job, {
cronConfig: opts?.cronConfig,
requireEnabled: patch.trigger !== null && patch.trigger !== undefined,
});
assertScriptPayloadSupport(job, {
cronConfig: opts?.cronConfig,
requireEnabled: patch.payload?.kind === "script",
// Enabled-only/rename patches must keep working on jobs stored with a
// malformed script (pre-validation persistence); re-check syntax only
// when this patch rewrites the payload, or disable becomes a dead end.
validateSyntax: patch.payload !== undefined,
});
assertStreamScheduleSupport(job, {
cronConfig: opts?.cronConfig,
requireEnabled: patch.enabled === true || patch.schedule?.kind === "stream",
});
assertMainSessionAgentId(job, opts?.defaultAgentId);
assertDeliverySupport(job);
assertAnnounceDeliveryChannelSupport(job, opts?.configuredChannels, patch);
assertFailureDestinationSupport(job);
if (
opts?.scheduleValidationNowMs !== undefined &&
(patch.schedule !== undefined || patch.enabled === true)
) {
assertCronExpressionSatisfiable(job, opts.scheduleValidationNowMs, computeJobNextRunAtMs);
}
validateFullJob(
job,
{
kind: "patch",
patch,
defaultAgentId: opts?.defaultAgentId,
nowMs: opts?.scheduleValidationNowMs,
cronConfig: opts?.cronConfig,
},
opts?.configuredChannels,
);
}
/** Converges the declared schedule, payload, delivery, and display label only. */
@@ -438,44 +481,19 @@ export function applyDeclarativeJobSpec(
const previousToolsAllowIsDefault = job.payload.toolsAllowIsDefault;
// Name, target, routing, owner, and run policy remain outside declaration
// convergence; changing those uses cron.update and cannot retarget an identity.
const displayName = normalizeOptionalString(input.displayName);
if (input.displayName !== undefined && !displayName) {
throw new Error("cron displayName must not be blank");
}
if (displayName && displayName.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
throw new Error(
`cron displayName must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
);
}
const displayName = normalizeDeclarativeLabel(input.displayName, "displayName");
if (displayName) {
job.displayName = displayName;
} else {
delete job.displayName;
}
if (
input.schedule.kind === "every" &&
input.schedule.anchorMs === undefined &&
job.schedule.kind === "every" &&
job.schedule.everyMs === input.schedule.everyMs
) {
job.schedule = { ...input.schedule, anchorMs: job.schedule.anchorMs };
} else if (input.schedule.kind === "every" && input.schedule.anchorMs === undefined) {
job.schedule = { ...input.schedule, anchorMs: opts.nowMs };
} else if (input.schedule.kind === "cron") {
const explicitStaggerMs = normalizeCronStaggerMs(input.schedule.staggerMs);
const defaultStaggerMs = resolveDefaultCronStaggerMs(input.schedule.expr);
job.schedule = {
...input.schedule,
...(explicitStaggerMs !== undefined
? { staggerMs: explicitStaggerMs }
: defaultStaggerMs !== undefined
? { staggerMs: defaultStaggerMs }
: {}),
};
} else {
job.schedule = normalizeStreamScheduleBounds(structuredClone(input.schedule));
}
job.schedule = normalizeJobSchedule(input.schedule, {
kind: "declarative",
previous: job.schedule,
nowMs: opts.nowMs,
fallbackAnchorMs: job.createdAtMs,
});
if (input.pacing !== undefined) {
job.pacing = structuredClone(input.pacing);
} else {
@@ -523,26 +541,17 @@ export function applyDeclarativeJobSpec(
if (opts.enabledExplicit) {
job.enabled = input.enabled;
}
assertTriggerSupport(job, {
cronConfig: opts.cronConfig,
requireEnabled: input.trigger !== undefined,
});
assertScriptPayloadSupport(job, {
cronConfig: opts.cronConfig,
requireEnabled: input.payload.kind === "script",
});
assertStreamScheduleSupport(job, {
cronConfig: opts.cronConfig,
requireEnabled: true,
});
assertSupportedJobSpec(job);
assertPacingSupport(job);
assertMainSessionAgentId(job, opts.defaultAgentId);
assertDeliverySupport(job);
assertAnnounceDeliveryChannelSupport(job, opts.configuredChannels);
assertFailureDestinationSupport(job);
assertCronExpressionSatisfiable(job, opts.nowMs, computeJobNextRunAtMs);
validateFullJob(
job,
{
kind: "declarative",
input,
defaultAgentId: opts.defaultAgentId,
nowMs: opts.nowMs,
cronConfig: opts.cronConfig,
},
opts.configuredChannels,
);
}
function mergeCronDelivery(
+16 -10
View File
@@ -1,5 +1,5 @@
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
import { nextWakeAtMs, recomputeNextRunsForMaintenance } from "./jobs.js";
import { nextWakeAtMs, recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
import { locked } from "./locked.js";
import { emitCronRunFinished } from "./ops-run-preparation.js";
import { cancelCronRunAdmissionWaiters } from "./run-admission.js";
@@ -59,16 +59,22 @@ export async function start(state: CronServiceState) {
...(finalized.triggerEval ? { triggerEval: finalized.triggerEval } : {}),
deferredNotifications: postPersistNotifications,
});
// Skip only the old invocation; a distinct overdue replacement
// must remain eligible for normal one-shot startup catch-up.
if (repaired.replacementAtMs === undefined) {
interruptedJobIds.add(job.id);
if (repaired) {
// Skip only the old invocation; a distinct overdue replacement
// must remain eligible for normal one-shot startup catch-up.
if (repaired.replacementAtMs === undefined) {
interruptedJobIds.add(job.id);
}
if (repaired.shouldDelete) {
completedJobIdsToDelete.add(job.id);
}
repairedAnyStartupRun = true;
continue;
}
if (repaired.shouldDelete) {
completedJobIdsToDelete.add(job.id);
}
repairedAnyStartupRun = true;
continue;
state.deps.log.warn(
{ jobId: job.id },
"cron: treating invalid finalized startup run as interrupted",
);
}
const nowMs = state.deps.nowMs();
const interrupted = markInterruptedStartupRun({
+3 -5
View File
@@ -18,18 +18,16 @@ import { removeStaleCronJobFamilyRows } from "../store.js";
import { createCronStreamSourceIdentity, cronStreamScheduleKey } from "../stream-schedule.js";
import { normalizeCronTaskRunJobId } from "../task-run-history.js";
import type { CronJob, CronJobCreate, CronJobPatch, CronStoredJob } from "../types.js";
import { cronPatchTouchesDeliveryResolution } from "./jobs-validation.js";
import {
applyJobPatch,
applyDeclarativeJobSpec,
computeJobNextRunAtMs,
createJob,
findJobOrThrow,
hasScheduledNextRunAtMs,
isJobEnabled,
nextWakeAtMs,
recomputeNextRunsForMaintenance,
} from "./jobs.js";
} from "./jobs-scheduling.js";
import { cronPatchTouchesDeliveryResolution } from "./jobs-validation.js";
import { applyJobPatch, applyDeclarativeJobSpec, createJob } from "./jobs.js";
import {
getPendingCronSessionCleanup,
locked,
+8 -1
View File
@@ -1,11 +1,17 @@
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
import { resolveOpenClawStateSqlitePath } from "../../state/openclaw-state-db.paths.js";
import { resolveCronListSnapshotRevision } from "../list-snapshot-revision.js";
import { assertCronJobStateTimestamps } from "../persisted-shape.js";
import { readCronJobScratchState, writeCronJobScratch } from "../scratch-store.js";
import { createCronStreamSourceIdentity } from "../stream-schedule.js";
import type { CronJob } from "../types.js";
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
import { findJobOrThrow, isJobEnabled, nextWakeAtMs, resolveJobLastRunStatus } from "./jobs.js";
import {
findJobOrThrow,
isJobEnabled,
nextWakeAtMs,
resolveJobLastRunStatus,
} from "./jobs-scheduling.js";
import { sortCronJobs } from "./list-page-sort.js";
import type {
CronJobsEnabledFilter,
@@ -113,6 +119,7 @@ export async function recordExternalFailure(
const postPersistNotifications: DeferredCronNotifications = [];
const now = state.deps.nowMs();
const sourceIdentity = job.state.streamSourceIdentity;
assertCronJobStateTimestamps(statePatch);
Object.assign(job.state, statePatch);
job.state.streamSourceIdentity = sourceIdentity;
// Source restarts are counted separately, but terminal exhaustion should
+8 -18
View File
@@ -5,17 +5,18 @@ import type { CronJob, CronPayload, CronRunErrorClassification } from "../types.
import { normalizeCronRunErrorText } from "./execution-errors.js";
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
import {
assertSupportedJobSpec,
findJobOrThrow,
hasActiveCronRun,
isJobDue,
isJobEnabled,
recomputeNextRunsForMaintenance,
} from "./jobs.js";
} from "./jobs-scheduling.js";
import { assertSupportedJobSpec } from "./jobs-validation.js";
import { locked } from "./locked.js";
import { markManualCronJobActive, ownsStreamSource } from "./ops-shared.js";
import {
activateQueuedCronRun,
cleanupQueuedCronRunReservations,
clearQueuedCronRunReservationMarker,
isQueuedCronRunReservationCurrent,
isQueuedCronRunReservationMarkerCurrent,
@@ -559,20 +560,9 @@ export async function releasePreparedManualReservationAfterReloadWithRetry(
state: CronServiceState,
prepared: Extract<PreparedManualRun, { ran: true }>,
): Promise<void> {
const attempt = async () => {
await locked(state, async () => {
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
await releasePreparedManualReservation(state, prepared);
});
};
try {
await attempt();
} catch {
try {
await attempt();
} catch (error) {
releaseQueuedCronRun(state, prepared.jobId, prepared.reservationIdentity);
throw error;
}
}
await cleanupQueuedCronRunReservations({
state,
reservations: [prepared],
restoreLastError: false,
});
}
+1 -1
View File
@@ -4,7 +4,7 @@ import { CommandLane } from "../../process/lanes.js";
import { isCronActiveJobMarkerCurrent } from "../active-jobs.js";
import { normalizeCronRunErrorText } from "./execution-errors.js";
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
import { recomputeNextRunsForMaintenance } from "./jobs.js";
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
import { locked } from "./locked.js";
import {
activatePreparedManualRun,
+1 -1
View File
@@ -3,7 +3,7 @@ import { parseAgentSessionKey } from "../../routing/session-key.js";
import { clearCronJobActive, markCronJobActive, type CronActiveJobMarker } from "../active-jobs.js";
import { cronStreamScheduleKey } from "../stream-schedule.js";
import type { CronJob } from "../types.js";
import { recomputeNextRunsForMaintenance } from "./jobs.js";
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
import { normalizeOptionalAgentId } from "./normalize.js";
import type { CronServiceState, DeferredCronNotifications } from "./state.js";
import { ensureLoaded, persist } from "./store.js";
+1 -1
View File
@@ -17,7 +17,7 @@ import { CommandLane } from "../../process/lanes.js";
import * as cronStoreModule from "../store.js";
import { loadCronStore, saveCronStore } from "../store.js";
import { cronStreamScheduleKey } from "../stream-schedule.js";
import { recomputeNextRunsForMaintenance } from "./jobs.js";
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
import { stop } from "./ops-lifecycle.js";
import { update } from "./ops-mutations.js";
import { list } from "./ops-read.js";
+225 -7
View File
@@ -1,8 +1,19 @@
// Shared execution admission for scheduled, manual, and on-exit cron runs.
import { DEFAULT_CRON_MAX_CONCURRENT_RUNS } from "../../config/cron-limits.js";
import { markCronJobActive } from "../active-jobs.js";
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
import type { CronJob } from "../types.js";
import type { CronServiceState } from "./state.js";
import { persistOrRestore, snapshotStoreForRollback } from "./store.js";
import { normalizeCronRunErrorText } from "./execution-errors.js";
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
import { locked } from "./locked.js";
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
import { ensureLoaded, persistOrRestore, snapshotStoreForRollback } from "./store.js";
import { tryCreateCronTaskRun } from "./task-runs.js";
import {
runsDetachedFromMainSession,
type TimedCronRunOutcome,
} from "./timer-execution-timeout.js";
import { executeJobCoreWithTimeout } from "./timer-job-runner.js";
import { isRunnableJob } from "./timer-runnable.js";
export function resolveRunConcurrency(): number {
return DEFAULT_CRON_MAX_CONCURRENT_RUNS;
@@ -95,7 +106,7 @@ export function isQueuedCronRunReservationCurrent(
return state.queuedRunReservationsByJobId.get(jobId)?.identity === identity;
}
export function restoreQueuedCronRunReservationLastError(
function restoreQueuedCronRunReservationLastError(
state: CronServiceState,
jobId: string,
identity: object,
@@ -113,6 +124,7 @@ export function clearQueuedCronRunReservationMarker(
jobId: string,
identity: object,
jobState: { queuedAtMs?: number; runningAtMs?: number; lastError?: string },
opts?: { restoreLastError?: boolean },
): boolean {
const reservation = state.queuedRunReservationsByJobId.get(jobId);
if (reservation?.identity !== identity) {
@@ -123,7 +135,9 @@ export function clearQueuedCronRunReservationMarker(
if (!queuedMatches && !runningMatches) {
return false;
}
restoreQueuedCronRunReservationLastError(state, jobId, identity, jobState);
if (opts?.restoreLastError !== false) {
restoreQueuedCronRunReservationLastError(state, jobId, identity, jobState);
}
if (queuedMatches) {
delete jobState.queuedAtMs;
}
@@ -133,6 +147,85 @@ export function clearQueuedCronRunReservationMarker(
return true;
}
type QueuedCronRunReservation = {
jobId: string;
reservationIdentity: object;
};
export function clearOwnedQueuedCronRunMarkers(
state: CronServiceState,
reservations: readonly QueuedCronRunReservation[],
opts?: { restoreLastError?: boolean },
): QueuedCronRunReservation[] {
const pendingReleases: QueuedCronRunReservation[] = [];
for (const reservation of reservations) {
const job = state.store?.jobs.find((entry) => entry.id === reservation.jobId);
if (
job &&
clearQueuedCronRunReservationMarker(
state,
reservation.jobId,
reservation.reservationIdentity,
job.state,
opts,
)
) {
pendingReleases.push(reservation);
} else {
releaseQueuedCronRun(state, reservation.jobId, reservation.reservationIdentity);
}
}
return pendingReleases;
}
/** Durably clears reservations still owned by this process. Ownership stays
* held through commit; after one retry it is dropped for restart repair. */
export async function cleanupQueuedCronRunReservations(params: {
state: CronServiceState;
reservations: readonly QueuedCronRunReservation[];
restoreLastError?: boolean;
recompute?: "maintenance" | "startup-overflow";
}): Promise<void> {
const { state, reservations } = params;
const attempt = async () => {
await locked(state, async () => {
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
const rollbackSnapshot = snapshotStoreForRollback(state);
const pendingReleases = clearOwnedQueuedCronRunMarkers(state, reservations, {
restoreLastError: params.restoreLastError,
});
if (pendingReleases.length === 0) {
return;
}
const postPersistNotifications: DeferredCronNotifications = [];
if (params.recompute) {
recomputeNextRunsForMaintenance(state, {
...(params.recompute === "startup-overflow"
? { repairFutureCronNextRunAtMs: false }
: {}),
deferredNotifications: postPersistNotifications,
});
}
await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications });
for (const reservation of pendingReleases) {
releaseQueuedCronRun(state, reservation.jobId, reservation.reservationIdentity);
}
});
};
try {
await attempt();
} catch {
try {
await attempt();
} catch (error) {
for (const reservation of reservations) {
releaseQueuedCronRun(state, reservation.jobId, reservation.reservationIdentity);
}
throw error;
}
}
}
export function isQueuedCronRunReservationMarkerCurrent(
state: CronServiceState,
jobId: string,
@@ -189,8 +282,7 @@ export async function activateQueuedCronRun(params: {
};
}
/**
* Apply one service-level cap to every cron execution source. Queue waiters
/** Apply one service-level cap to every cron execution source. Queue waiters
* keep their job reservation, then recheck scheduler state before execution.
*/
export async function runWithCronAdmission<T>(
@@ -207,3 +299,129 @@ export async function runWithCronAdmission<T>(
release();
}
}
export async function executeQueuedCronRun(params: {
state: CronServiceState;
jobId: string;
reservedAtMs: number;
reservationIdentity: object;
runnableOptions?: Omit<Parameters<typeof isRunnableJob>[0], "state" | "job" | "nowMs">;
isUnavailable?: () => boolean;
onUnavailable?: () => void;
onActivated?: () => void;
onNotRunnable: (job: CronJob) => Promise<void>;
onSetupError?: (job: CronJob, errorText: string) => void;
/** Runs before admission release; true means terminal handling is complete. */
onCompleted?: (outcome: TimedCronRunOutcome) => Promise<boolean>;
}): Promise<
| { kind: "stopped" }
| { kind: "skipped" }
| { kind: "completed"; outcome: TimedCronRunOutcome; handled: boolean }
> {
const { state } = params;
let activated = false;
const admission = await runWithCronAdmission(state, async () => {
const started = await locked(state, async () => {
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
if (params.isUnavailable?.() || state.stopped || state.restartRecoveryPending) {
params.onUnavailable?.();
return undefined;
}
const job = state.store?.jobs.find((entry) => entry.id === params.jobId);
if (
!job ||
!isQueuedCronRunReservationCurrent(state, params.jobId, params.reservationIdentity) ||
job.state.queuedAtMs !== params.reservedAtMs
) {
releaseQueuedCronRun(state, params.jobId, params.reservationIdentity);
return undefined;
}
const runnableJob = structuredClone(job);
delete runnableJob.state.queuedAtMs;
if (
!isRunnableJob({
state,
job: runnableJob,
nowMs: state.deps.nowMs(),
...params.runnableOptions,
})
) {
await params.onNotRunnable(job);
return undefined;
}
const activation = await activateQueuedCronRun({
state,
job,
reservationIdentity: params.reservationIdentity,
onUnavailable: params.onUnavailable,
});
if (activation.kind !== "activated") {
return undefined;
}
activated = true;
params.onActivated?.();
return { job, startedAt: activation.startedAt };
});
if (!started) {
return undefined;
}
const executionJob = structuredClone(started.job);
executionJob.state.runningAtMs = started.startedAt;
executionJob.state.lastError = undefined;
const taskRunId = tryCreateCronTaskRun({
state,
job: executionJob,
startedAt: started.startedAt,
});
const activeJobMarker = markCronJobActive(executionJob.id, {
preserveAcrossGenerationAdvance: !runsDetachedFromMainSession(executionJob),
});
emit(state, {
jobId: executionJob.id,
action: "started",
job: executionJob,
runAtMs: started.startedAt,
});
const base = {
jobId: params.jobId,
job: executionJob,
taskRunId,
activeJobMarker,
reservationIdentity: params.reservationIdentity,
startedAt: started.startedAt,
};
let outcome: TimedCronRunOutcome;
try {
const result = await executeJobCoreWithTimeout(state, executionJob, {
runId: taskRunId,
activeJobMarker,
});
outcome = { ...base, ...result, endedAt: state.deps.nowMs() };
} catch (error) {
const errorText = normalizeCronRunErrorText(error);
params.onSetupError?.(executionJob, errorText);
outcome = {
...base,
status: "error",
error: errorText,
diagnostics: createCronRunDiagnosticsFromError("cron-setup", errorText, {
nowMs: state.deps.nowMs,
}),
endedAt: state.deps.nowMs(),
};
}
return { outcome, handled: (await params.onCompleted?.(outcome)) === true };
}).catch((error: unknown) => {
if (activated) {
releaseQueuedCronRun(state, params.jobId, params.reservationIdentity);
}
throw error;
});
if (admission.kind === "stopped") {
return { kind: "stopped" };
}
if (!admission.value) {
return { kind: "skipped" };
}
return { kind: "completed", ...admission.value };
}
@@ -1,6 +1,7 @@
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it, vi } from "vitest";
import type { CronJob } from "../types.js";
import { markInterruptedStartupRun } from "./startup-run-repair.js";
import { markInterruptedStartupRun, restoreFinalizedStartupRun } from "./startup-run-repair.js";
import { createCronServiceState } from "./state.js";
describe("startup run repair auto-disable", () => {
@@ -68,4 +69,117 @@ describe("startup run repair auto-disable", () => {
expect(enqueueSystemEvent).toHaveBeenCalledOnce();
expect(requestHeartbeat).toHaveBeenCalledOnce();
});
it("disables a job instead of restoring an invalid finalized next run", () => {
const runningAtMs = Date.parse("2026-08-01T16:00:00.000Z");
const state = createCronServiceState({
storePath: "/tmp/startup-run-repair-invalid-next-run.json",
cronEnabled: true,
log: {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
},
nowMs: () => runningAtMs + 1_000,
enqueueSystemEvent: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(),
});
const job: CronJob = {
id: "invalid-finalized-next-run",
name: "invalid finalized next run",
enabled: true,
createdAtMs: runningAtMs - 60_000,
updatedAtMs: runningAtMs,
schedule: { kind: "every", everyMs: 60_000, anchorMs: runningAtMs - 60_000 },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: "do not replay" },
state: { nextRunAtMs: runningAtMs, runningAtMs },
};
const deferredNotifications: Array<() => void> = [];
restoreFinalizedStartupRun({
state,
job,
runningAtMs,
deferredNotifications,
entry: {
ts: runningAtMs + 1_000,
jobId: job.id,
action: "finished",
status: "ok",
runAtMs: runningAtMs,
durationMs: 1_000,
nextRunAtMs: MAX_DATE_TIMESTAMP_MS + 1,
},
});
expect(job.enabled).toBe(false);
expect(job.state.nextRunAtMs).toBeUndefined();
expect(job.state.autoDisabled).toEqual({
reason: "schedule-errors",
atMs: runningAtMs + 1_000,
consecutiveErrors: 1,
});
expect(state.deps.enqueueSystemEvent).not.toHaveBeenCalled();
expect(state.deps.requestHeartbeat).not.toHaveBeenCalled();
expect(deferredNotifications).toHaveLength(1);
deferredNotifications[0]?.();
expect(state.deps.enqueueSystemEvent).toHaveBeenCalledOnce();
expect(state.deps.requestHeartbeat).toHaveBeenCalledOnce();
});
it.each(["runAtMs", "ts"] as const)(
"ignores finalized startup history with an invalid %s",
(field) => {
const runningAtMs = Date.parse("2026-08-01T16:00:00.000Z");
const warn = vi.fn();
const state = createCronServiceState({
storePath: "/tmp/startup-run-repair-invalid-history.json",
cronEnabled: true,
log: { debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() },
nowMs: () => runningAtMs + 1_000,
enqueueSystemEvent: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(),
});
const job: CronJob = {
id: "invalid-finalized-history",
name: "invalid finalized history",
enabled: true,
createdAtMs: runningAtMs - 60_000,
updatedAtMs: runningAtMs,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: "do not replay" },
state: { nextRunAtMs: runningAtMs, runningAtMs },
};
const before = structuredClone(job);
const result = restoreFinalizedStartupRun({
state,
job,
runningAtMs,
entry: {
ts: field === "ts" ? MAX_DATE_TIMESTAMP_MS + 1 : runningAtMs + 1_000,
jobId: job.id,
action: "finished",
status: "ok",
runAtMs: field === "runAtMs" ? MAX_DATE_TIMESTAMP_MS + 1 : runningAtMs,
durationMs: 1_000,
},
});
expect(result).toBeUndefined();
expect(job).toEqual(before);
expect(warn).toHaveBeenCalledWith(
{ jobId: job.id },
"cron: ignoring finalized startup run with an invalid timestamp envelope",
);
},
);
});
+26 -9
View File
@@ -1,10 +1,12 @@
/** Repairs interrupted and finalized cron runs while the service starts. */
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
import { resolveCronDeliveryPlan, resolveFailureDestination } from "../delivery-plan.js";
import { parseAbsoluteTimeMs } from "../parse.js";
import type { CronRunLogEntry } from "../run-log-types.js";
import type { CronJob, CronRunStatus } from "../types.js";
import { maybeAutoDisableCronJobAfterRunFailure } from "./auto-disable.js";
import type { CronServiceState, DeferredCronNotifications } from "./state.js";
import { resolveNextRunAtMsOrDisable } from "./timer-trigger.js";
import {
applyJobResult,
applyScriptRunResult,
@@ -127,9 +129,17 @@ export function restoreFinalizedStartupRun(params: {
scriptResult?: { scriptStateChanged: true; scriptState?: unknown };
triggerEval?: CronTriggerEvalOutcome;
deferredNotifications?: DeferredCronNotifications;
}): { shouldDelete: boolean; replacementAtMs?: number } {
}): { shouldDelete: boolean; replacementAtMs?: number } | undefined {
const { state, job, runningAtMs, entry } = params;
const startedAt = entry.runAtMs ?? runningAtMs;
const startedAt = asDateTimestampMs(entry.runAtMs ?? runningAtMs);
const endedAt = asDateTimestampMs(entry.ts);
if (startedAt === undefined || startedAt < 0 || endedAt === undefined || endedAt < 0) {
state.deps.log.warn(
{ jobId: job.id },
"cron: ignoring finalized startup run with an invalid timestamp envelope",
);
return undefined;
}
const replacementAtMs = resolveOneShotReplacementAtMs(job, startedAt);
const scheduleOwnership = replacementAtMs === undefined ? "current" : "stale";
const shouldDelete = applyJobResult(
@@ -138,17 +148,17 @@ export function restoreFinalizedStartupRun(params: {
{
...entry,
startedAt,
endedAt: entry.ts,
endedAt,
},
{
replayFailureAlertAtMs: entry.ts,
replayFailureAlertAtMs: endedAt,
scheduleOwnership,
deferredNotifications: params.deferredNotifications,
},
);
// The finalized row captured post-run state before the stale cron store write.
job.state.lastDurationMs = entry.durationMs ?? Math.max(0, entry.ts - startedAt);
job.state.lastDurationMs = entry.durationMs ?? Math.max(0, endedAt - startedAt);
job.state.lastErrorReason = entry.errorReason;
job.state.lastDelivered = entry.delivered;
job.state.lastDeliveryStatus = entry.deliveryStatus;
@@ -156,9 +166,16 @@ export function restoreFinalizedStartupRun(params: {
job.state.lastFailureNotificationDelivered = entry.failureNotificationDelivery?.delivered;
job.state.lastFailureNotificationDeliveryStatus = entry.failureNotificationDelivery?.status;
job.state.lastFailureNotificationDeliveryError = entry.failureNotificationDelivery?.error;
job.state.nextRunAtMs = job.state.autoDisabled
? undefined
: (replacementAtMs ?? entry.nextRunAtMs);
const finalizedNextRunAtMs = replacementAtMs ?? entry.nextRunAtMs;
job.state.nextRunAtMs =
job.state.autoDisabled || finalizedNextRunAtMs === undefined
? undefined
: resolveNextRunAtMsOrDisable({
state,
job,
candidate: finalizedNextRunAtMs,
deferredNotifications: params.deferredNotifications,
});
// The finalized ledger row owns the schedule decision made before the stale
// store write. No next run means that one-shot was permanently disabled.
if (
@@ -173,7 +190,7 @@ export function restoreFinalizedStartupRun(params: {
job,
{
status: entry.status,
endedAt: entry.ts,
endedAt,
triggerEval: params.triggerEval,
},
{ scheduleOwnership },
@@ -1,6 +1,6 @@
// Missing session target tests cover loading legacy cron state without targets.
import { describe, expect, it } from "vitest";
import { assertSupportedJobSpec } from "./jobs.js";
import { assertSupportedJobSpec } from "./jobs-validation.js";
describe("cron service store load: missing sessionTarget", () => {
it("assertSupportedJobSpec throws a clear error when sessionTarget is missing", () => {
+94 -1
View File
@@ -1,12 +1,13 @@
// Cron service store tests cover persisted service state loading and writes.
import fs from "node:fs/promises";
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { afterEach, describe, expect, it, vi } from "vitest";
import { openOpenClawStateDatabase } from "../../state/openclaw-state-db.js";
import { setupCronServiceSuite } from "../service.test-harness.js";
import * as cronStoreModule from "../store.js";
import { loadCronStore, saveCronStore } from "../store.js";
import type { CronJob } from "../types.js";
import { findJobOrThrow } from "./jobs.js";
import { findJobOrThrow } from "./jobs-scheduling.js";
import { createCronServiceState } from "./state.js";
import { ensureLoaded, persist, persistOrRestore, snapshotStoreForRollback } from "./store.js";
@@ -153,6 +154,98 @@ describe("cron service store seam coverage", () => {
await expectPathMissing(storePath.replace(/\.json$/, "-quarantine.json"));
});
it("quarantines persisted every schedules that cannot produce valid Date timestamps", async () => {
const { storePath } = await makeStorePath();
const invalidInterval = createReloadCronJob({
id: "invalid-date-interval",
schedule: { kind: "every", everyMs: 1_000 },
});
const invalidAnchor = createReloadCronJob({
id: "invalid-date-anchor",
schedule: { kind: "every", everyMs: 1_000, anchorMs: 0 },
});
const unsatisfiableInterval = createReloadCronJob({
id: "unsatisfiable-date-interval",
schedule: { kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS },
});
const disabledUnsatisfiableInterval = createReloadCronJob({
id: "disabled-unsatisfiable-date-interval",
enabled: false,
schedule: { kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS },
});
const invalidStagger = createReloadCronJob({ id: "invalid-date-stagger" });
const repairableState = createReloadCronJob({
id: "repairable-runtime-state",
state: { lastRunAtMs: MAX_DATE_TIMESTAMP_MS },
});
const surviving = createReloadCronJob({ id: "valid-schedule" });
await saveCronStore(storePath, {
version: 1,
jobs: [
invalidInterval,
invalidAnchor,
unsatisfiableInterval,
disabledUnsatisfiableInterval,
invalidStagger,
repairableState,
surviving,
],
});
const db = openOpenClawStateDatabase().db;
db.prepare("UPDATE cron_jobs SET every_ms = ? WHERE job_id = ?").run(
MAX_DATE_TIMESTAMP_MS + 1,
invalidInterval.id,
);
db.prepare("UPDATE cron_jobs SET anchor_ms = ? WHERE job_id = ?").run(
MAX_DATE_TIMESTAMP_MS + 1,
invalidAnchor.id,
);
db.prepare("UPDATE cron_jobs SET stagger_ms = ? WHERE job_id = ?").run(
MAX_DATE_TIMESTAMP_MS + 1,
invalidStagger.id,
);
const state = createStoreTestState(storePath);
await ensureLoaded(state, { skipRecompute: true });
expect(state.store?.jobs.map((job) => job.id)).toEqual([
disabledUnsatisfiableInterval.id,
repairableState.id,
surviving.id,
]);
expect(cronStoreModule.loadCronQuarantinedJobs(storePath)).toEqual([
expect.objectContaining({ job: expect.objectContaining({ id: invalidInterval.id }) }),
expect.objectContaining({ job: expect.objectContaining({ id: invalidAnchor.id }) }),
expect.objectContaining({
reason: "unsatisfiable-schedule",
job: expect.objectContaining({ id: unsatisfiableInterval.id }),
}),
expect.objectContaining({ job: expect.objectContaining({ id: invalidStagger.id }) }),
]);
});
it("quarantines persisted runtime timestamps outside the Date domain", async () => {
const { storePath } = await makeStorePath();
const invalidState = createReloadCronJob({ id: "invalid-runtime-state" });
const surviving = createReloadCronJob({ id: "valid-runtime-state" });
await saveCronStore(storePath, { version: 1, jobs: [invalidState, surviving] });
openOpenClawStateDatabase()
.db.prepare("UPDATE cron_jobs SET last_run_at_ms = ? WHERE job_id = ?")
.run(MAX_DATE_TIMESTAMP_MS + 1, invalidState.id);
const state = createStoreTestState(storePath);
await ensureLoaded(state, { skipRecompute: true });
expect(state.store?.jobs.map((job) => job.id)).toEqual([surviving.id]);
expect(cronStoreModule.loadCronQuarantinedJobs(storePath)).toEqual([
expect.objectContaining({
reason: "invalid-state",
job: expect.objectContaining({ id: invalidState.id }),
state: expect.objectContaining({ lastRunAtMs: MAX_DATE_TIMESTAMP_MS + 1 }),
}),
]);
});
it("publishes durable wake changes only after save and exactly once after retry", async () => {
const { storePath } = await makeStorePath();
const initialNextRunAtMs = STORE_TEST_NOW + 60_000;
+18 -3
View File
@@ -12,7 +12,8 @@ import {
type QuarantinedCronConfigJob,
} from "../store.js";
import type { CronJob, CronStoreFile } from "../types.js";
import { recomputeNextRuns } from "./jobs.js";
import { computeJobNextRunAtMs, recomputeNextRuns } from "./jobs-scheduling.js";
import { assertTimeScheduleSatisfiable } from "./jobs-validation.js";
import { emit, type CronServiceState, type DeferredCronNotifications } from "./state.js";
const loadedCronStoreRevisions = new WeakMap<CronServiceState, number>();
@@ -142,6 +143,7 @@ export async function ensureLoaded(
previousJobsById.set(job.id, job);
}
const loaded = await loadCronJobsStoreWithConfigJobs(state.deps.storePath);
const loadNowMs = state.deps.nowMs();
// Persisted cron rows are validated lazily, so treat them as raw records at the
// store boundary and only trust the CronJob shape after validation below.
const loadedJobs = (loaded.store.jobs ?? []) as unknown as Record<string, unknown>[];
@@ -155,6 +157,7 @@ export async function ensureLoaded(
// Accept old `jobId` rows at the raw boundary only; the in-memory store
// uses canonical `id` before validation and scheduling.
normalizeCronJobIdentityFields(raw);
const rawInvalidReason = getInvalidPersistedCronJobReason(raw);
let normalized: Record<string, unknown> | null;
try {
normalized = normalizeCronJobInput(raw);
@@ -169,7 +172,19 @@ export async function ensureLoaded(
);
}
const hydratedRaw = normalized ?? raw;
const invalidReason = getInvalidPersistedCronJobReason(hydratedRaw);
let invalidReason = rawInvalidReason ?? getInvalidPersistedCronJobReason(hydratedRaw);
const hydratedSchedule = (hydratedRaw.schedule ?? {}) as Record<string, unknown>;
if (!invalidReason && hydratedRaw.enabled !== false && hydratedSchedule.kind === "every") {
try {
assertTimeScheduleSatisfiable(
{ ...(hydratedRaw as unknown as CronJob), state: {} },
loadNowMs,
computeJobNextRunAtMs,
);
} catch {
invalidReason = "unsatisfiable-schedule";
}
}
if (invalidReason) {
const quarantineEntry: QuarantinedCronConfigJob = {
sourceIndex,
@@ -206,7 +221,7 @@ export async function ensureLoaded(
jobs,
};
state.durableNextRunAtMsByJobId = durableNextRunAtMsByJobId;
state.storeLoadedAtMs = state.deps.nowMs();
state.storeLoadedAtMs = loadNowMs;
loadedCronStoreRevisions.set(state, getCronJobsStoreRevision(state.deps.storePath));
if (quarantinedConfigJobs.length > 0) {
+57 -164
View File
@@ -1,45 +1,33 @@
import { markCronJobActive } from "../active-jobs.js";
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
import { normalizeCronRunErrorText } from "./execution-errors.js";
import {
DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
isJobEnabled,
recomputeNextRunsForMaintenance,
resolveJobErrorBackoffUntilMs,
} from "./jobs.js";
} from "./jobs-scheduling.js";
import { locked } from "./locked.js";
import {
activateQueuedCronRun,
isQueuedCronRunReservationCurrent,
clearOwnedQueuedCronRunMarkers,
cleanupQueuedCronRunReservations,
executeQueuedCronRun,
releaseQueuedCronRun,
reserveQueuedCronRun,
runWithCronAdmission,
} from "./run-admission.js";
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
import type { CronServiceState, DeferredCronNotifications } from "./state.js";
import { ensureLoaded, persist, persistOrRestore, snapshotStoreForRollback } from "./store.js";
import { tryCreateCronTaskRun } from "./task-runs.js";
import {
DEFAULT_MAX_MISSED_JOBS_PER_RESTART,
DEFAULT_MISSED_JOB_STAGGER_MS,
DEFAULT_STARTUP_DEFERRED_MISSED_AGENT_JOB_DELAY_MS,
runsDetachedFromMainSession,
type StartupCatchupCandidate,
type StartupCatchupExecution,
type StartupCatchupPlan,
type StartupDeferredJob,
type TimedCronRunOutcome,
} from "./timer-execution-timeout.js";
import { executeJobCoreWithTimeout } from "./timer-job-runner.js";
import {
clearUnstartedStartupCatchupReservationMarkers,
createCompletedCronRunOutcomeDrain,
} from "./timer-outcome-finalization.js";
import {
collectRunnableJobs,
hasMissedCronSlotSinceLastRun,
isRunnableJob,
} from "./timer-runnable.js";
import { createCompletedCronRunOutcomeDrain } from "./timer-outcome-finalization.js";
import { collectRunnableJobs, hasMissedCronSlotSinceLastRun } from "./timer-runnable.js";
import { maybeNotifyIsolatedAgentSetupTimeout } from "./timer-scheduler.js";
import { resolveNextRunAtMsOrDisable } from "./timer-trigger.js";
function deferPendingBackoffMissedCronSlots(
state: CronServiceState,
@@ -79,8 +67,8 @@ async function persistStartupCatchupReservations(
state: CronServiceState,
rollbackSnapshot: ReturnType<typeof snapshotStoreForRollback>,
pendingReleases: readonly Pick<StartupCatchupCandidate, "jobId" | "reservationIdentity">[],
postPersistNotifications: DeferredCronNotifications = [],
): Promise<void> {
const postPersistNotifications: DeferredCronNotifications = [];
recomputeNextRunsForMaintenance(state, {
repairFutureCronNextRunAtMs: false,
deferredNotifications: postPersistNotifications,
@@ -97,31 +85,12 @@ async function releaseStartupCatchupReservationsAfterFailure(
plan: StartupCatchupPlan,
outcomes: readonly TimedCronRunOutcome[],
): Promise<void> {
const attempt = async () => {
await locked(state, async () => {
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
const rollbackSnapshot = snapshotStoreForRollback(state);
const pendingReleases = clearUnstartedStartupCatchupReservationMarkers(state, plan, outcomes);
if (pendingReleases.length === 0) {
return;
}
await persistStartupCatchupReservations(state, rollbackSnapshot, pendingReleases);
});
};
try {
await attempt();
} catch {
try {
await attempt();
} catch (error) {
// The failed execution has no remaining cleanup owner. Release process
// claims so durable stuck-marker recovery can eventually repair them.
for (const candidate of plan.candidates) {
releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity);
}
throw error;
}
}
const startedJobIds = new Set(outcomes.map((outcome) => outcome.jobId));
await cleanupQueuedCronRunReservations({
state,
reservations: plan.candidates.filter((candidate) => !startedJobIds.has(candidate.jobId)),
recompute: "startup-overflow",
});
}
/** Runs or defers missed startup jobs using restart catch-up limits. */
@@ -281,69 +250,29 @@ async function executeStartupCatchupPlan(
if (state.stopped) {
break;
}
const admission = await runWithCronAdmission(state, async () => {
const startedCandidate = await locked(state, async () => {
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
const job = state.store?.jobs.find((entry) => entry.id === candidate.jobId);
if (state.stopped || state.restartRecoveryPending) {
return undefined;
}
if (
!job ||
!isQueuedCronRunReservationCurrent(
state,
candidate.jobId,
candidate.reservationIdentity,
) ||
job.state.queuedAtMs !== candidate.reservedAtMs
) {
releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity);
return undefined;
}
const dueProbe = structuredClone(job);
delete dueProbe.state.queuedAtMs;
if (
!isRunnableJob({
state,
job: dueProbe,
nowMs: state.deps.nowMs(),
skipAtIfAlreadyRan: true,
allowCronMissedRunByLastRun: true,
})
) {
const rollbackSnapshot = snapshotStoreForRollback(state);
delete job.state.queuedAtMs;
await persistStartupCatchupReservations(state, rollbackSnapshot, [candidate]);
return undefined;
}
const activation = await activateQueuedCronRun({
state,
job,
reservationIdentity: candidate.reservationIdentity,
});
if (activation.kind === "unavailable") {
return undefined;
}
return { ...candidate, job, startedAt: activation.startedAt };
});
if (!startedCandidate) {
return undefined;
}
try {
return await runStartupCatchupCandidate(state, startedCandidate);
} catch (error) {
releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity);
throw error;
}
const execution = await executeQueuedCronRun({
state,
jobId: candidate.jobId,
reservedAtMs: candidate.reservedAtMs,
reservationIdentity: candidate.reservationIdentity,
runnableOptions: {
skipAtIfAlreadyRan: true,
allowCronMissedRunByLastRun: true,
},
onNotRunnable: async (job) => {
const rollbackSnapshot = snapshotStoreForRollback(state);
delete job.state.queuedAtMs;
await persistStartupCatchupReservations(state, rollbackSnapshot, [candidate]);
},
});
if (admission.kind === "stopped") {
if (execution.kind === "stopped") {
break;
}
if (admission.value) {
if (execution.kind === "completed") {
// Catch-up execution stays sequential, while completed outcomes
// persist in coalesced batches before slower siblings have to drain.
outcomes.push(admission.value);
completedOutcomeDrain.enqueue(admission.value);
outcomes.push(execution.outcome);
completedOutcomeDrain.enqueue(execution.outcome);
}
}
} catch (error) {
@@ -352,62 +281,6 @@ async function executeStartupCatchupPlan(
return { ok: true, outcomes };
}
async function runStartupCatchupCandidate(
state: CronServiceState,
candidate: StartupCatchupCandidate & { startedAt: number },
): Promise<TimedCronRunOutcome> {
const { startedAt } = candidate;
const executionJob = structuredClone(candidate.job);
executionJob.state.runningAtMs = startedAt;
const taskRunId = tryCreateCronTaskRun({
state,
job: executionJob,
startedAt,
});
const activeJobMarker = markCronJobActive(executionJob.id, {
preserveAcrossGenerationAdvance: !runsDetachedFromMainSession(executionJob),
});
emit(state, {
jobId: executionJob.id,
action: "started",
job: executionJob,
runAtMs: startedAt,
});
try {
const result = await executeJobCoreWithTimeout(state, executionJob, {
runId: taskRunId,
activeJobMarker,
});
return {
jobId: candidate.jobId,
job: executionJob,
taskRunId,
activeJobMarker,
reservationIdentity: candidate.reservationIdentity,
// Keep the complete core outcome: startup catch-up shares the same result
// application path as timer runs, including delivery and script state.
...result,
startedAt,
endedAt: state.deps.nowMs(),
};
} catch (err) {
return {
jobId: candidate.jobId,
job: executionJob,
taskRunId,
activeJobMarker,
reservationIdentity: candidate.reservationIdentity,
status: "error",
error: normalizeCronRunErrorText(err),
diagnostics: createCronRunDiagnosticsFromError("cron-setup", normalizeCronRunErrorText(err), {
nowMs: state.deps.nowMs,
}),
startedAt,
endedAt: state.deps.nowMs(),
};
}
}
async function applyStartupCatchupOutcomes(
state: CronServiceState,
plan: StartupCatchupPlan,
@@ -422,7 +295,12 @@ async function applyStartupCatchupOutcomes(
return;
}
const rollbackSnapshot = snapshotStoreForRollback(state);
const pendingReleases = clearUnstartedStartupCatchupReservationMarkers(state, plan, outcomes);
const startedJobIds = new Set(outcomes.map((outcome) => outcome.jobId));
const pendingReleases = clearOwnedQueuedCronRunMarkers(
state,
plan.candidates.filter((candidate) => !startedJobIds.has(candidate.jobId)),
);
const postPersistNotifications: DeferredCronNotifications = [];
if (state.stopped || (outcomes.length === 0 && plan.deferredJobs.length === 0)) {
if (pendingReleases.length > 0) {
await persistStartupCatchupReservations(state, rollbackSnapshot, pendingReleases);
@@ -440,13 +318,23 @@ async function applyStartupCatchupOutcomes(
continue;
}
if (typeof deferred.delayMs === "number") {
const runAtMs = baseNow + deferred.delayMs + offset - staggerMs;
const runAtMs = resolveNextRunAtMsOrDisable({
state,
job,
candidate: baseNow + deferred.delayMs + offset - staggerMs,
deferredNotifications: postPersistNotifications,
});
job.state.nextRunAtMs = runAtMs;
job.state.startupCatchupAtMs = runAtMs;
offset += staggerMs;
continue;
}
const runAtMs = baseNow + offset;
const runAtMs = resolveNextRunAtMsOrDisable({
state,
job,
candidate: baseNow + offset,
deferredNotifications: postPersistNotifications,
});
job.state.nextRunAtMs = runAtMs;
job.state.startupCatchupAtMs = runAtMs;
offset += staggerMs;
@@ -455,7 +343,12 @@ async function applyStartupCatchupOutcomes(
// Startup overflow owns these staggered wake times; repairing future
// schedules here would silently move a deferred run to its natural slot.
await persistStartupCatchupReservations(state, rollbackSnapshot, pendingReleases);
await persistStartupCatchupReservations(
state,
rollbackSnapshot,
pendingReleases,
postPersistNotifications,
);
});
return outcomes;
}
+1 -1
View File
@@ -16,7 +16,7 @@ import type {
CronRunTelemetry,
} from "../types.js";
import { abortErrorMessage, timeoutErrorMessage } from "./execution-errors.js";
import { resolveJobPayloadTextForMain } from "./jobs.js";
import { resolveJobPayloadTextForMain } from "./jobs-scheduling.js";
import type { CronServiceState } from "./state.js";
import { resolveMainSessionCronRunSessionKey } from "./task-runs.js";
import {
+2 -35
View File
@@ -2,9 +2,9 @@
import { clearCronJobActive, isCronActiveJobMarkerCurrent } from "../active-jobs.js";
import type { CronActiveJobMarker } from "../active-jobs.js";
import type { CronJob } from "../types.js";
import { recomputeNextRunsForMaintenance } from "./jobs.js";
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
import { locked } from "./locked.js";
import { clearQueuedCronRunReservationMarker, releaseQueuedCronRun } from "./run-admission.js";
import { releaseQueuedCronRun } from "./run-admission.js";
import { emit, type CronServiceState, type DeferredCronNotifications } from "./state.js";
import {
ensureLoaded,
@@ -28,10 +28,6 @@ type CronTaskRunFinalizationOutcome = {
activeJobMarker?: CronActiveJobMarker;
};
type StartupCatchupReservationPlan = {
candidates: readonly { jobId: string; reservedAtMs: number; reservationIdentity: object }[];
};
type CompletedCronRunOutcomeFinalizationOptions = {
clearOnFailure?: boolean;
discardWhenStopped?: boolean;
@@ -224,32 +220,3 @@ function finishRetiredCronTaskRuns<T extends CronTaskRunFinalizationOutcome>(
}
}
}
export function clearUnstartedStartupCatchupReservationMarkers(
state: CronServiceState,
plan: StartupCatchupReservationPlan,
outcomes: readonly CronTaskRunFinalizationOutcome[],
): Array<{ jobId: string; reservationIdentity: object }> {
const pendingReleases: Array<{ jobId: string; reservationIdentity: object }> = [];
const startedJobIds = new Set(outcomes.map((outcome) => outcome.jobId));
for (const candidate of plan.candidates) {
if (startedJobIds.has(candidate.jobId)) {
continue;
}
const job = state.store?.jobs.find((entry) => entry.id === candidate.jobId);
if (
job &&
clearQueuedCronRunReservationMarker(
state,
candidate.jobId,
candidate.reservationIdentity,
job.state,
)
) {
pendingReleases.push(candidate);
} else {
releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity);
}
}
return pendingReleases;
}
+127 -107
View File
@@ -5,7 +5,6 @@ import { normalizeCronRunDiagnostics, summarizeCronRunDiagnostics } from "../run
import { resolveCronRunErrorReason } from "../run-error-reason.js";
import { cronSchedulingInputsEqual } from "../schedule-identity.js";
import { computeNextRunAtMs } from "../schedule.js";
import { createCronStreamSourceIdentity } from "../stream-schedule.js";
import type { CronJob, CronRunStatus } from "../types.js";
import { maybeAutoDisableCronJobAfterRunFailure } from "./auto-disable.js";
import {
@@ -19,7 +18,7 @@ import {
errorBackoffMs,
isJobEnabled,
recordScheduleComputeError,
} from "./jobs.js";
} from "./jobs-scheduling.js";
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
import { tryFinishCronTaskRun, tryFinishCronTaskRunWithoutHistory } from "./task-runs.js";
import {
@@ -29,9 +28,12 @@ import {
type TimedCronRunOutcome,
} from "./timer-execution-timeout.js";
import {
applyTriggerEvaluationState,
applyTriggerRunResult,
resolveCronNextRunWithLowerBound,
resolveDeliveryState,
resolveDisabledHeartbeatOneShotRetryDecision,
resolveNextRunAtMsOrDisable,
resolveTransientCronRetryDecision,
shouldRetryDisabledHeartbeatOneShot,
} from "./timer-trigger.js";
@@ -64,6 +66,14 @@ export function resolveCronRunTriggerOwnership(params: {
: "current";
}
function assignNextRunAtMs(
params: Parameters<typeof resolveNextRunAtMsOrDisable>[0],
): number | undefined {
const nextRunAtMs = resolveNextRunAtMsOrDisable(params);
params.job.state.nextRunAtMs = nextRunAtMs;
return nextRunAtMs;
}
/** Applies run outcome state, delivery state, backoff/next-run scheduling, and delete-after-run policy. */
export function applyJobResult(
state: CronServiceState,
@@ -219,17 +229,25 @@ export function applyJobResult(
});
if (retryDecision.retryable && retryDecision.backoffMs !== undefined) {
job.enabled = true;
job.state.nextRunAtMs = result.endedAt + retryDecision.backoffMs;
state.deps.log.info(
{
jobId: job.id,
jobName: job.name,
consecutiveSkipped: retryDecision.consecutiveSkipped,
backoffMs: retryDecision.backoffMs,
nextRunAtMs: job.state.nextRunAtMs,
},
"cron: scheduling one-shot retry after disabled heartbeat",
);
if (
assignNextRunAtMs({
state,
job,
candidate: result.endedAt + retryDecision.backoffMs,
deferredNotifications: opts?.deferredNotifications,
}) !== undefined
) {
state.deps.log.info(
{
jobId: job.id,
jobName: job.name,
consecutiveSkipped: retryDecision.consecutiveSkipped,
backoffMs: retryDecision.backoffMs,
nextRunAtMs: job.state.nextRunAtMs,
},
"cron: scheduling one-shot retry after disabled heartbeat",
);
}
} else {
job.enabled = false;
job.state.nextRunAtMs = undefined;
@@ -258,18 +276,26 @@ export function applyJobResult(
});
if (retryDecision.retryable && retryDecision.backoffMs !== undefined) {
// Schedule retry with backoff (#24355).
job.state.nextRunAtMs = result.endedAt + retryDecision.backoffMs;
state.deps.log.info(
{
jobId: job.id,
jobName: job.name,
consecutiveErrors: retryDecision.consecutiveErrors,
backoffMs: retryDecision.backoffMs,
nextRunAtMs: job.state.nextRunAtMs,
retryCategory: retryDecision.retryCategory,
},
"cron: scheduling one-shot retry after transient error",
);
if (
assignNextRunAtMs({
state,
job,
candidate: result.endedAt + retryDecision.backoffMs,
deferredNotifications: opts?.deferredNotifications,
}) !== undefined
) {
state.deps.log.info(
{
jobId: job.id,
jobName: job.name,
consecutiveErrors: retryDecision.consecutiveErrors,
backoffMs: retryDecision.backoffMs,
nextRunAtMs: job.state.nextRunAtMs,
retryCategory: retryDecision.retryCategory,
},
"cron: scheduling one-shot retry after transient error",
);
}
} else {
// Permanent error or max retries exhausted: disable.
// Note: deleteAfterRun:true only triggers on ok (see shouldDelete above),
@@ -354,25 +380,34 @@ export function applyJobResult(
};
if (retryDecision.retryable && retryDecision.backoffMs !== undefined) {
normalNext = computeNormalNext();
const retryNextRunAtMs = result.endedAt + retryDecision.backoffMs;
if (normalNext === undefined) {
// Preserve the unresolved-cron guard (#66019): do not synthesize a
// retry when the schedule cannot produce a next scheduled slot.
} else if (retryNextRunAtMs < normalNext) {
job.state.nextRunAtMs = retryNextRunAtMs;
state.deps.log.info(
{
jobId: job.id,
jobName: job.name,
consecutiveErrors: retryDecision.consecutiveErrors,
backoffMs: retryDecision.backoffMs,
nextRunAtMs: job.state.nextRunAtMs,
normalNextRunAtMs: normalNext,
retryCategory: retryDecision.retryCategory,
},
"cron: scheduling recurring retry after transient error",
);
return shouldDelete;
} else {
const retryNextRunAtMs = assignNextRunAtMs({
state,
job,
candidate: result.endedAt + retryDecision.backoffMs,
deferredNotifications: opts?.deferredNotifications,
});
if (retryNextRunAtMs === undefined) {
return shouldDelete;
}
if (retryNextRunAtMs < normalNext) {
state.deps.log.info(
{
jobId: job.id,
jobName: job.name,
consecutiveErrors: retryDecision.consecutiveErrors,
backoffMs: retryDecision.backoffMs,
nextRunAtMs: job.state.nextRunAtMs,
normalNextRunAtMs: normalNext,
retryCategory: retryDecision.retryCategory,
},
"cron: scheduling recurring retry after transient error",
);
return shouldDelete;
}
}
}
// Apply exponential backoff for errored jobs to prevent retry storms.
@@ -381,7 +416,24 @@ export function applyJobResult(
DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
);
normalNext = computeNormalNext();
const backoffNext = result.endedAt + backoff;
if (normalNext === undefined && job.schedule.kind === "every") {
assignNextRunAtMs({
state,
job,
candidate: undefined,
deferredNotifications: opts?.deferredNotifications,
});
return shouldDelete;
}
const backoffNext = assignNextRunAtMs({
state,
job,
candidate: result.endedAt + backoff,
deferredNotifications: opts?.deferredNotifications,
});
if (backoffNext === undefined) {
return shouldDelete;
}
// Use whichever is later: the natural next run or the backoff delay.
job.state.nextRunAtMs =
job.schedule.kind === "cron"
@@ -390,7 +442,7 @@ export function applyJobResult(
job,
naturalNext: normalNext,
lowerBoundMs: backoffNext,
context: "error_backoff",
deferredNotifications: opts?.deferredNotifications,
})
: normalNext !== undefined
? Math.max(normalNext, backoffNext)
@@ -419,13 +471,17 @@ export function applyJobResult(
});
// The operator trigger floor is a safety policy and outranks a job-local
// pacing bound. Non-trigger jobs retain the exact pacing clamp contract.
const nextRunAtMs = job.trigger
? Math.max(
pacedNextRunAtMs,
result.endedAt + Math.max(MIN_REFIRE_GAP_MS, resolveCronTriggerMinIntervalMs()),
)
: pacedNextRunAtMs;
job.state.nextRunAtMs = nextRunAtMs;
const nextRunAtMs = assignNextRunAtMs({
state,
job,
candidate: job.trigger
? Math.max(
pacedNextRunAtMs ?? Number.NaN,
result.endedAt + Math.max(MIN_REFIRE_GAP_MS, resolveCronTriggerMinIntervalMs()),
)
: pacedNextRunAtMs,
deferredNotifications: opts?.deferredNotifications,
});
job.state.pacedNextRunAtMs = nextRunAtMs;
} else if (isJobEnabled(job)) {
let naturalNext: number | undefined;
@@ -460,13 +516,22 @@ export function applyJobResult(
job,
naturalNext,
lowerBoundMs: minNext,
context: "completion",
deferredNotifications: opts?.deferredNotifications,
});
} else {
job.state.nextRunAtMs =
const triggerNext =
naturalNext !== undefined && job.trigger
? Math.max(naturalNext, result.endedAt + resolveCronTriggerMinIntervalMs())
: naturalNext;
job.state.nextRunAtMs = triggerNext;
if (triggerNext !== undefined || job.schedule.kind === "every") {
assignNextRunAtMs({
state,
job,
candidate: triggerNext,
deferredNotifications: opts?.deferredNotifications,
});
}
}
} else {
job.state.nextRunAtMs = undefined;
@@ -476,59 +541,6 @@ export function applyJobResult(
return shouldDelete;
}
function applyTriggerEvaluationState(
job: CronJob,
triggerEval: CronTriggerEvalOutcome,
evaluatedAtMs: number,
): void {
if (triggerEval.busy) {
return;
}
job.state.lastTriggerEvalAtMs = evaluatedAtMs;
job.state.triggerEvalCount = (job.state.triggerEvalCount ?? 0) + 1;
if (triggerEval.stateChanged) {
job.state.triggerState = triggerEval.state;
}
if (triggerEval.fired) {
job.state.lastTriggerFireAtMs = evaluatedAtMs;
}
}
/** Persists fired/error evaluation metadata and applies successful once-disarm policy. */
export function applyTriggerRunResult(
job: CronJob,
result: { status: CronRunStatus; endedAt: number; triggerEval?: CronTriggerEvalOutcome },
opts?: { scheduleOwnership?: CronScheduleOwnership; triggerOwnership?: CronTriggerOwnership },
): void {
if (!result.triggerEval || opts?.triggerOwnership === "stale") {
return;
}
// Fired-run trigger state persists only on payload success: a failed or
// skipped run keeps the previous state so the next evaluation re-detects
// the change and fires again instead of silently losing the event.
const persistedEval =
result.status === "ok"
? result.triggerEval
: { ...result.triggerEval, stateChanged: false, state: undefined };
applyTriggerEvaluationState(job, persistedEval, result.endedAt);
// A once trigger disarms only after the fired payload succeeds. Errors keep
// it armed so the normal backoff path can evaluate and retry later.
if (
opts?.scheduleOwnership !== "stale" &&
result.triggerEval.fired &&
job.trigger?.once === true &&
result.status === "ok"
) {
if (job.schedule.kind === "stream") {
// Auto-disable is a source retirement just like an explicit disable. Rotate
// in the same persisted result so queued sibling batches cannot gain admission.
job.state.streamSourceIdentity = createCronStreamSourceIdentity();
}
job.enabled = false;
job.state.nextRunAtMs = undefined;
}
}
/** Commits payload-script state only after the complete cron run succeeds. */
export function applyScriptRunResult(
job: CronJob,
@@ -591,8 +603,16 @@ export function applyTriggerNoFireResult(
const floorMs = Math.max(MIN_REFIRE_GAP_MS, resolveCronTriggerMinIntervalMs());
// Quiet ticks still advance the schedule; the floor prevents scripts from
// becoming a headless hot loop even when cron resolves inside the window.
job.state.nextRunAtMs =
naturalNext === undefined ? undefined : Math.max(naturalNext, result.endedAt + floorMs);
job.state.nextRunAtMs = naturalNext;
if (naturalNext !== undefined || job.schedule.kind === "every") {
assignNextRunAtMs({
state,
job,
candidate:
naturalNext === undefined ? undefined : Math.max(naturalNext, result.endedAt + floorMs),
deferredNotifications: opts?.deferredNotifications,
});
}
} catch (err) {
recordScheduleComputeError({
state,
+1 -1
View File
@@ -8,7 +8,7 @@ import {
isJobEnabled,
resolveJobErrorBackoffUntilMs,
resolveJobLastRunStatus,
} from "./jobs.js";
} from "./jobs-scheduling.js";
import type { CronServiceState } from "./state.js";
import { isScheduledTerminalOneShotRetry } from "./timer-trigger.js";
+79 -263
View File
@@ -5,46 +5,35 @@ import {
GatewayDrainingError,
} from "../../process/gateway-work-admission.js";
import { normalizeAgentId, resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
import { markCronJobActive } from "../active-jobs.js";
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
import { sweepCronRunSessions } from "../session-reaper.js";
import type { CronJob } from "../types.js";
import { normalizeCronRunErrorText } from "./execution-errors.js";
import {
hasScheduledNextRunAtMs,
isJobEnabled,
nextWakeAtMs,
recomputeNextRunsForMaintenance,
} from "./jobs.js";
} from "./jobs-scheduling.js";
import { locked } from "./locked.js";
import {
activateQueuedCronRun,
clearQueuedCronRunReservationMarker,
isQueuedCronRunReservationCurrent,
isQueuedCronRunReservationMarkerCurrent,
cleanupQueuedCronRunReservations,
executeQueuedCronRun,
releaseQueuedCronRun,
reserveQueuedCronRun,
resolveRunConcurrency,
restoreQueuedCronRunReservationLastError,
runWithCronAdmission,
} from "./run-admission.js";
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
import type { CronServiceState, DeferredCronNotifications } from "./state.js";
import { ensureLoaded, persistOrRestore, snapshotStoreForRollback } from "./store.js";
import { tryCreateCronTaskRun } from "./task-runs.js";
import { resolveCronJobTimeoutMs } from "./timeout-policy.js";
import {
MAX_TIMER_DELAY_MS,
MIN_REFIRE_GAP_MS,
runsDetachedFromMainSession,
type TimedCronRunOutcome,
} from "./timer-execution-timeout.js";
import { executeJobCoreWithTimeout } from "./timer-job-runner.js";
import { maybeNotifyIsolatedAgentSetupTimeoutWithRecovery } from "./timer-notifications.js";
import {
createCompletedCronRunOutcomeDrain,
finalizeCompletedCronRunOutcomes,
} from "./timer-outcome-finalization.js";
import { collectRunnableJobs, isRunnableJob } from "./timer-runnable.js";
import { collectRunnableJobs } from "./timer-runnable.js";
export function maybeNotifyIsolatedAgentSetupTimeout(
state: CronServiceState,
@@ -233,130 +222,9 @@ async function onAdmittedTimer(state: CronServiceState) {
reservedAtMs: now,
reservationIdentity: reserveQueuedCronRun(state, job.id, now),
}));
if (state.stopped) {
const cleanup = async () => {
const rollbackSnapshot = snapshotStoreForRollback(state);
const pendingReleases: typeof reservedDue = [];
for (const candidate of reservedDue) {
if (
!isQueuedCronRunReservationCurrent(state, candidate.id, candidate.reservationIdentity)
) {
continue;
}
const persistedJob = state.store?.jobs.find((entry) => entry.id === candidate.id);
if (
typeof persistedJob?.state.queuedAtMs === "number" &&
isQueuedCronRunReservationMarkerCurrent(
state,
candidate.id,
candidate.reservationIdentity,
persistedJob.state.queuedAtMs,
)
) {
restoreQueuedCronRunReservationLastError(
state,
candidate.id,
candidate.reservationIdentity,
persistedJob.state,
);
delete persistedJob.state.queuedAtMs;
pendingReleases.push(candidate);
} else {
releaseQueuedCronRun(state, candidate.id, candidate.reservationIdentity);
}
}
const postPersistNotifications: DeferredCronNotifications = [];
recomputeNextRunsForMaintenance(state, {
deferredNotifications: postPersistNotifications,
});
await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications });
for (const candidate of pendingReleases) {
releaseQueuedCronRun(state, candidate.id, candidate.reservationIdentity);
}
};
try {
await cleanup();
} catch {
try {
await cleanup();
} catch (error) {
// The stopped scheduler has no later cleanup pass.
for (const candidate of reservedDue) {
releaseQueuedCronRun(state, candidate.id, candidate.reservationIdentity);
}
throw error;
}
}
return [];
}
return reservedDue;
});
const runDueJob = async (params: {
id: string;
job: CronJob;
reservationIdentity: object;
startedAt: number;
}): Promise<TimedCronRunOutcome> => {
const { id, job, startedAt } = params;
const executionJob = structuredClone(job);
executionJob.state.runningAtMs = startedAt;
executionJob.state.lastError = undefined;
const activeJobMarker = markCronJobActive(executionJob.id, {
preserveAcrossGenerationAdvance: !runsDetachedFromMainSession(executionJob),
});
emit(state, {
jobId: executionJob.id,
action: "started",
job: executionJob,
runAtMs: startedAt,
});
const jobTimeoutMs = resolveCronJobTimeoutMs(executionJob);
const taskRunId = tryCreateCronTaskRun({
state,
job: executionJob,
startedAt,
});
try {
const result = await executeJobCoreWithTimeout(state, executionJob, {
runId: taskRunId,
activeJobMarker,
});
return {
jobId: id,
job: executionJob,
taskRunId,
activeJobMarker,
reservationIdentity: params.reservationIdentity,
...result,
startedAt,
endedAt: state.deps.nowMs(),
};
} catch (err) {
const errorText = normalizeCronRunErrorText(err);
state.deps.log.warn(
{ jobId: id, jobName: executionJob.name, timeoutMs: jobTimeoutMs ?? null },
`cron: job failed: ${errorText}`,
);
return {
jobId: id,
job: executionJob,
taskRunId,
activeJobMarker,
reservationIdentity: params.reservationIdentity,
status: "error",
error: errorText,
diagnostics: createCronRunDiagnosticsFromError("cron-setup", errorText, {
nowMs: state.deps.nowMs,
}),
startedAt,
endedAt: state.deps.nowMs(),
};
}
};
const concurrency = Math.min(resolveRunConcurrency(), Math.max(1, dueJobs.length));
const completedOutcomeDrain = createCompletedCronRunOutcomeDrain(state);
const claimedIndexes = new Set<number>();
@@ -364,53 +232,18 @@ async function onAdmittedTimer(state: CronServiceState) {
let setupTimeoutNotified = false;
let stopAdmittingDueJobs = false;
const hasSetupTimeoutRecoveryHandler = state.deps.onIsolatedAgentSetupTimeout !== undefined;
const releaseUnclaimedDueJobReservations = async () => {
if (claimedIndexes.size >= dueJobs.length) {
return;
}
await locked(state, async () => {
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
const rollbackSnapshot = snapshotStoreForRollback(state);
const pendingReleases: typeof dueJobs = [];
for (const [index, due] of dueJobs.entries()) {
if (claimedIndexes.has(index)) {
continue;
}
const job = state.store?.jobs.find((entry) => entry.id === due.id);
if (
job &&
clearQueuedCronRunReservationMarker(state, due.id, due.reservationIdentity, job.state)
) {
pendingReleases.push(due);
} else {
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
}
}
const postPersistNotifications: DeferredCronNotifications = [];
recomputeNextRunsForMaintenance(state, { deferredNotifications: postPersistNotifications });
await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications });
for (const due of pendingReleases) {
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
}
});
};
const releaseUnclaimedDueJobReservationsWithRetry = async () => {
try {
await releaseUnclaimedDueJobReservations();
} catch {
try {
await releaseUnclaimedDueJobReservations();
} catch (error) {
// No timer task owns another retry. Drop only these process claims so
// durable stuck-marker recovery remains able to repair them.
for (const [index, due] of dueJobs.entries()) {
if (!claimedIndexes.has(index)) {
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
}
}
throw error;
}
}
const reservations = dueJobs
.filter((_, index) => !claimedIndexes.has(index))
.map((due) => ({
jobId: due.id,
reservationIdentity: due.reservationIdentity,
}));
await cleanupQueuedCronRunReservations({
state,
reservations,
recompute: "maintenance",
});
};
if (state.stopped) {
await releaseUnclaimedDueJobReservationsWithRetry();
@@ -429,92 +262,75 @@ async function onAdmittedTimer(state: CronServiceState) {
return pMapSkip;
}
try {
const admission = await runWithCronAdmission(state, async () => {
const currentDueJob = await locked(state, async () => {
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
if (stopAdmittingDueJobs || state.stopped || state.restartRecoveryPending) {
stopAdmittingDueJobs = true;
return undefined;
}
const job = state.store?.jobs.find((entry) => entry.id === due.id);
if (
!job ||
!isQueuedCronRunReservationCurrent(state, due.id, due.reservationIdentity) ||
job.state.queuedAtMs !== due.reservedAtMs
) {
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
return undefined;
}
const dueProbe = structuredClone(job);
delete dueProbe.state.queuedAtMs;
if (
!isJobEnabled(job) ||
!isRunnableJob({ state, job: dueProbe, nowMs: state.deps.nowMs() })
) {
const rollbackSnapshot = snapshotStoreForRollback(state);
delete job.state.queuedAtMs;
await persistOrRestore(state, rollbackSnapshot);
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
return undefined;
}
const activation = await activateQueuedCronRun({
state,
job,
reservationIdentity: due.reservationIdentity,
onUnavailable: () => {
stopAdmittingDueJobs = true;
},
});
if (activation.kind === "unavailable") {
return undefined;
}
return { ...due, job, startedAt: activation.startedAt };
});
if (!currentDueJob) {
return pMapSkip;
}
claimedIndexes.add(index);
let result: TimedCronRunOutcome;
try {
result = await runDueJob(currentDueJob);
} catch (error) {
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
throw error;
}
if (!result.isolatedAgentSetupTimeout) {
// Drain finished state independently: a slow sibling must not
// strand outcomes, and store I/O must not own execution slots.
completedOutcomeDrain.enqueue(result);
return pMapSkip;
}
let finalizedResults: TimedCronRunOutcome[];
try {
finalizedResults = await finalizeCompletedCronRunOutcomes(state, [result], {
clearOnFailure: false,
});
} catch {
return result;
}
if (!hasSetupTimeoutRecoveryHandler || finalizedResults.length === 0) {
return pMapSkip;
}
if (!setupTimeoutNotified) {
setupTimeoutNotified = true;
const execution = await executeQueuedCronRun({
state,
jobId: due.id,
reservedAtMs: due.reservedAtMs,
reservationIdentity: due.reservationIdentity,
isUnavailable: () => stopAdmittingDueJobs,
onUnavailable: () => {
stopAdmittingDueJobs = true;
try {
await releaseUnclaimedDueJobReservationsWithRetry();
} catch (err) {
reservationReleaseError = err;
},
onActivated: () => claimedIndexes.add(index),
onNotRunnable: async (job) => {
const rollbackSnapshot = snapshotStoreForRollback(state);
delete job.state.queuedAtMs;
await persistOrRestore(state, rollbackSnapshot);
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
},
onSetupError: (job, errorText) => {
state.deps.log.warn(
{
jobId: due.id,
jobName: job.name,
timeoutMs: resolveCronJobTimeoutMs(job) ?? null,
},
`cron: job failed: ${errorText}`,
);
},
onCompleted: async (result) => {
if (!result.isolatedAgentSetupTimeout) {
// Drain finished state independently: a slow sibling must not
// strand outcomes, and store I/O must not own execution slots.
completedOutcomeDrain.enqueue(result);
return true;
}
maybeNotifyIsolatedAgentSetupTimeout(state, result);
}
return pMapSkip;
let finalizedResults: TimedCronRunOutcome[];
try {
finalizedResults = await finalizeCompletedCronRunOutcomes(state, [result], {
clearOnFailure: false,
});
} catch {
return false;
}
if (
hasSetupTimeoutRecoveryHandler &&
finalizedResults.length > 0 &&
!setupTimeoutNotified
) {
setupTimeoutNotified = true;
stopAdmittingDueJobs = true;
try {
await releaseUnclaimedDueJobReservationsWithRetry();
} catch (err) {
reservationReleaseError = err;
}
maybeNotifyIsolatedAgentSetupTimeout(state, result);
}
return true;
},
});
if (admission.kind === "stopped") {
if (execution.kind === "stopped") {
stopAdmittingDueJobs = true;
return pMapSkip;
}
return admission.value;
if (execution.kind === "skipped") {
return pMapSkip;
}
if (execution.handled) {
return pMapSkip;
}
return execution.outcome;
} catch (error) {
stopAdmittingDueJobs = true;
batchExecutionError ??= error;
+92 -5
View File
@@ -1,6 +1,8 @@
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
import type { CronConfig } from "../../config/types.cron.js";
import { resolveCronDeliveryPlan, resolveFailureDestination } from "../delivery-plan.js";
import { type CronRetryOn, resolveCronExecutionRetryHint } from "../retry-hint.js";
import { createCronStreamSourceIdentity } from "../stream-schedule.js";
import type {
CronDeliveryStatus,
CronFailureNotificationDelivery,
@@ -8,8 +10,18 @@ import type {
CronRunErrorClassification,
CronRunStatus,
} from "../types.js";
import { DEFAULT_ERROR_BACKOFF_SCHEDULE_MS, errorBackoffMs, isJobEnabled } from "./jobs.js";
import type { CronServiceState, CronSystemEventEnqueueResult } from "./state.js";
import { autoDisableCronJob } from "./auto-disable.js";
import {
DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
errorBackoffMs,
isJobEnabled,
} from "./jobs-scheduling.js";
import type {
CronServiceState,
CronSystemEventEnqueueResult,
DeferredCronNotifications,
} from "./state.js";
import type { CronTriggerEvalOutcome } from "./timer-execution-timeout.js";
import { HEARTBEAT_SKIP_DISABLED } from "./timer-execution-timeout.js";
/** Default max retries for cron jobs on transient errors (#24355). */
@@ -35,25 +47,100 @@ type QueuedSystemEventHandle = {
remove?: () => boolean | void;
};
/** Rejects outcome-generated schedule timestamps before they can persist or arm a timer. */
export function resolveNextRunAtMsOrDisable(params: {
state: CronServiceState;
job: CronJob;
candidate: unknown;
deferredNotifications?: DeferredCronNotifications;
}): number | undefined {
const nextRunAtMs = asDateTimestampMs(params.candidate);
if (nextRunAtMs !== undefined && nextRunAtMs > 0) {
return nextRunAtMs;
}
autoDisableCronJob({
state: params.state,
job: params.job,
reason: "schedule-errors",
atMs: params.state.deps.nowMs(),
consecutiveErrors: 1,
error: "next run is outside the supported Date range",
deferredNotifications: params.deferredNotifications,
});
return undefined;
}
/** Persists non-busy trigger evaluation state without touching payload-run history. */
export function applyTriggerEvaluationState(
job: CronJob,
triggerEval: CronTriggerEvalOutcome,
evaluatedAtMs: number,
): void {
if (triggerEval.busy) {
return;
}
job.state.lastTriggerEvalAtMs = evaluatedAtMs;
job.state.triggerEvalCount = (job.state.triggerEvalCount ?? 0) + 1;
if (triggerEval.stateChanged) {
job.state.triggerState = triggerEval.state;
}
if (triggerEval.fired) {
job.state.lastTriggerFireAtMs = evaluatedAtMs;
}
}
/** Persists fired/error trigger metadata and disarms successful once triggers. */
export function applyTriggerRunResult(
job: CronJob,
result: { status: CronRunStatus; endedAt: number; triggerEval?: CronTriggerEvalOutcome },
opts?: { scheduleOwnership?: "current" | "stale"; triggerOwnership?: "current" | "stale" },
): void {
if (!result.triggerEval || opts?.triggerOwnership === "stale") {
return;
}
// Failed payloads keep the old state so the next evaluation re-detects the event.
const persistedEval =
result.status === "ok"
? result.triggerEval
: { ...result.triggerEval, stateChanged: false, state: undefined };
applyTriggerEvaluationState(job, persistedEval, result.endedAt);
if (
opts?.scheduleOwnership !== "stale" &&
result.triggerEval.fired &&
job.trigger?.once === true &&
result.status === "ok"
) {
if (job.schedule.kind === "stream") {
job.state.streamSourceIdentity = createCronStreamSourceIdentity();
}
job.enabled = false;
job.state.nextRunAtMs = undefined;
}
}
export function resolveCronNextRunWithLowerBound(params: {
state: CronServiceState;
job: CronJob;
naturalNext: number | undefined;
lowerBoundMs: number;
context: "completion" | "error_backoff";
deferredNotifications?: DeferredCronNotifications;
}): number | undefined {
if (params.naturalNext === undefined) {
params.state.deps.log.warn(
{
jobId: params.job.id,
jobName: params.job.name,
context: params.context,
},
"cron: next run unresolved; clearing schedule to avoid a refire loop",
);
return undefined;
}
return Math.max(params.naturalNext, params.lowerBoundMs);
return resolveNextRunAtMsOrDisable({
state: params.state,
job: params.job,
candidate: Math.max(params.naturalNext, params.lowerBoundMs),
deferredNotifications: params.deferredNotifications,
});
}
export function resolveTransientCronRetryDecision(params: {
@@ -1,4 +1,5 @@
// Completed cron work must become durable before unrelated batch work drains.
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
createDueIsolatedJob,
@@ -427,6 +428,84 @@ describe("cron batch outcome finalization", () => {
}
});
it("records and notifies a Date-overflow auto-disable only after persistence", async () => {
const store = fixtures.makeStorePath();
const dueAt = Date.parse("2026-08-01T15:02:00.000Z");
const job = createDueIsolatedJob({
id: "date-overflow-auto-disable",
nowMs: dueAt,
nextRunAtMs: dueAt,
});
job.schedule = { kind: "every", everyMs: 60_000, anchorMs: dueAt - 60_000 };
job.pacing = { min: "1s" };
job.state.runningAtMs = dueAt;
await saveCronStore(store.storePath, { version: 1, jobs: [job] });
const order: string[] = [];
const enqueueSystemEvent = vi.fn((_text: string) => {
order.push("notify");
});
const requestHeartbeat = vi.fn(() => {
order.push("heartbeat");
});
const state = createCronServiceState({
cronEnabled: true,
storePath: store.storePath,
log: noopLogger,
nowMs: () => dueAt + 10,
enqueueSystemEvent,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(),
});
const save = cronStoreModule.saveCronJobsStore;
const saveSpy = vi
.spyOn(cronStoreModule, "saveCronJobsStore")
.mockImplementation(async (...args) => {
if (args[1].jobs[0]?.state.autoDisabled) {
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
order.push("persist");
}
return await save(...args);
});
try {
const finalized = await finalizeCompletedCronRunOutcomes(state, [
{
jobId: job.id,
job: structuredClone(job),
activeJobMarker: markCronJobActive(job.id),
status: "ok",
startedAt: dueAt,
endedAt: dueAt + 10,
nextCheck: { delayMs: MAX_DATE_TIMESTAMP_MS },
},
]);
expect(finalized).toHaveLength(1);
expect(state.store?.jobs[0]?.enabled).toBe(false);
expect(state.store?.jobs[0]?.state.nextRunAtMs).toBeUndefined();
expect(order).toEqual(["persist", "notify", "heartbeat"]);
expect(enqueueSystemEvent).toHaveBeenCalledOnce();
expect(enqueueSystemEvent.mock.calls[0]?.[0]).toContain(
"next run is outside the supported Date range",
);
expect(requestHeartbeat).toHaveBeenCalledOnce();
expect((await loadCronStore(store.storePath)).jobs[0]).toMatchObject({
enabled: false,
state: {
autoDisabled: {
reason: "schedule-errors",
atMs: dueAt + 10,
consecutiveErrors: 1,
},
},
});
} finally {
saveSpy.mockRestore();
}
});
it("rolls back recurring auto-disable without notifying when persistence fails", async () => {
const store = fixtures.makeStorePath();
const dueAt = Date.parse("2026-08-01T15:05:00.000Z");
+91 -1
View File
@@ -1,3 +1,4 @@
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it, vi } from "vitest";
import {
clearCronJobActive,
@@ -7,7 +8,7 @@ import {
import { makeCronJob } from "../delivery.test-helpers.js";
import { createNoopLogger } from "../service.test-harness.js";
import type { CronJob, CronPacing } from "../types.js";
import { recomputeNextRunsForMaintenance } from "./jobs.js";
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
import { createCronServiceState } from "./state.js";
import { applyOutcomeToStoredJob, applyTriggerNoFireResult } from "./timer-outcomes.js";
import { applyJobResult } from "./timer.js";
@@ -120,6 +121,95 @@ describe("cron trigger evaluation ownership", () => {
});
describe("applyJobResult dynamic cadence", () => {
it.each(["one-shot retry", "recurring retry", "pacing", "trigger floor", "quiet trigger"])(
"auto-disables a job when %s cannot produce a Date-valid next run",
(scenario) => {
const endedAt = MAX_DATE_TIMESTAMP_MS - 1_000;
const state = makeState();
const deferredNotifications: Array<() => void> = [];
const job = makeCronJob({
schedule:
scenario === "one-shot retry"
? { kind: "at", at: new Date(endedAt).toISOString() }
: { kind: "every", everyMs: 1_000, anchorMs: 0 },
state: { nextRunAtMs: endedAt },
...(scenario === "pacing" ? { pacing: { min: "1s" } } : {}),
...(scenario === "trigger floor" || scenario === "quiet trigger"
? { trigger: { script: "return true" } }
: {}),
});
if (scenario === "quiet trigger") {
applyTriggerNoFireResult(
state,
job,
{
startedAt: endedAt - 1,
endedAt,
triggerEval: { fired: false, stateChanged: false },
},
{ deferredNotifications },
);
} else {
const isRetry = scenario === "one-shot retry" || scenario === "recurring retry";
applyJobResult(
state,
job,
{
status: isRetry ? "error" : "ok",
...(isRetry
? {
error: "temporary timeout",
errorClassification: { kind: "reason" as const, reason: "timeout" as const },
executionStarted: true,
}
: {}),
startedAt: endedAt - 1,
endedAt,
...(scenario === "pacing" ? { nextCheck: { delayMs: 2_000 } } : {}),
},
{ deferredNotifications },
);
}
expect(job.enabled).toBe(false);
expect(job.state.nextRunAtMs).toBeUndefined();
expect(job.state.pacedNextRunAtMs).toBeUndefined();
expect(job.state.autoDisabled).toEqual({
reason: "schedule-errors",
atMs: ENDED_AT,
consecutiveErrors: 1,
});
expect(state.deps.enqueueSystemEvent).not.toHaveBeenCalled();
expect(state.deps.requestHeartbeat).not.toHaveBeenCalled();
expect(deferredNotifications).toHaveLength(1);
deferredNotifications[0]?.();
expect(state.deps.enqueueSystemEvent).toHaveBeenCalledOnce();
expect(state.deps.requestHeartbeat).toHaveBeenCalledOnce();
},
);
it("disables an exhausted every schedule instead of synthesizing a backoff-only run", () => {
const endedAt = MAX_DATE_TIMESTAMP_MS - 39_000;
const job = makeCronJob({
schedule: { kind: "every", everyMs: 60_000, anchorMs: 0 },
state: { nextRunAtMs: endedAt },
});
applyJobResult(makeState(), job, {
status: "error",
error: "permanent failure",
errorClassification: { kind: "permanent" },
startedAt: endedAt - 1_000,
endedAt,
});
expect(endedAt + 30_000).toBeLessThanOrEqual(MAX_DATE_TIMESTAMP_MS);
expect(job.enabled).toBe(false);
expect(job.state.nextRunAtMs).toBeUndefined();
});
it.each([
["honors an in-range proposal", { min: "15m", max: "4h" }, 60 * 60_000, 60 * 60_000],
["clamps below the minimum", { min: "15m", max: "4h" }, 5 * 60_000, 15 * 60_000],
+69 -1
View File
@@ -33,7 +33,7 @@ import { readCronTaskRunHistoryPage } from "../task-run-history.js";
import type { CronAgentExecutionPhaseUpdate, CronJob } from "../types.js";
import { cancelActiveCronTaskRun } from "./active-run-cancellation.js";
import { resetActiveCronTaskRunsForTests } from "./active-run-cancellation.test-support.js";
import { computeJobNextRunAtMs, recomputeNextRunsForMaintenance } from "./jobs.js";
import { computeJobNextRunAtMs, recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
import { stop } from "./ops-lifecycle.js";
import { run as runManualCronJob } from "./ops-run.js";
import { createCronServiceState as createBaseCronServiceState, type CronEvent } from "./state.js";
@@ -2152,6 +2152,74 @@ describe("cron service timer regressions", () => {
}
});
it("does not release a running sibling when setup-timeout recovery clears queued jobs", async () => {
vi.useFakeTimers();
try {
const store = timerRegressionFixtures.makeStorePath();
const dueAt = Date.parse("2026-02-06T10:06:21.000Z");
const stalled = createDueIsolatedJob({
id: "setup-timeout-stalled",
nowMs: dueAt,
nextRunAtMs: dueAt,
});
const running = createDueIsolatedJob({
id: "setup-timeout-running-sibling",
nowMs: dueAt,
nextRunAtMs: dueAt,
});
stalled.payload = { kind: "agentTurn", message: "stall", timeoutSeconds: 120 };
running.payload = { kind: "agentTurn", message: "run", timeoutSeconds: 120 };
await saveCronStore(store.storePath, { version: 1, jobs: [stalled, running] });
let now = dueAt;
const runningStarted = createDeferred();
const finishRunning = createDeferred<{ status: "ok"; summary: string }>();
const timeoutNotified = createDeferred();
const state = createCronServiceState({
cronEnabled: true,
storePath: store.storePath,
testAdmissionLimit: 2,
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeat: vi.fn(),
onIsolatedAgentSetupTimeout: () => timeoutNotified.resolve(),
runIsolatedAgentJob: vi.fn(
async ({
job,
onExecutionStarted,
}: Parameters<CronStateParams["runIsolatedAgentJob"]>[0]) => {
if (job.id === stalled.id) {
return await new Promise<never>(() => {});
}
onExecutionStarted?.({ jobId: job.id, phase: "model_call_started" });
runningStarted.resolve();
return await finishRunning.promise;
},
),
});
const timerPromise = onTimer(state);
await runningStarted.promise;
await vi.advanceTimersByTimeAsync(60_100);
now += 60_100;
await timeoutNotified.promise;
const runningAtMsAfterRecovery = requireJob(state, running.id).state.runningAtMs;
const reservationHeldAfterRecovery = state.queuedRunReservationsByJobId.has(running.id);
finishRunning.resolve({ status: "ok", summary: "finished" });
await timerPromise;
expect({ runningAtMsAfterRecovery, reservationHeldAfterRecovery }).toEqual({
runningAtMsAfterRecovery: dueAt,
reservationHeldAfterRecovery: true,
});
expect(requireJob(state, running.id).state.lastStatus).toBe("ok");
} finally {
vi.useRealTimers();
}
});
it("notifies timeout recovery before admitting queued manual work", async () => {
vi.useFakeTimers();
try {
+1 -1
View File
@@ -7,7 +7,7 @@ export { runsDetachedFromMainSession } from "./timer-execution-timeout.js";
export { executeJobCoreWithTimeout } from "./timer-job-runner.js";
export { maybeNotifyIsolatedAgentSetupTimeout } from "./timer-scheduler.js";
export { applyJobResult } from "./timer-outcomes.js";
export { applyTriggerRunResult } from "./timer-outcomes.js";
export { applyTriggerRunResult } from "./timer-trigger.js";
export { applyScriptRunResult } from "./timer-outcomes.js";
export { applyTriggerNoFireResult } from "./timer-outcomes.js";
export { armTimer } from "./timer-scheduler.js";
+2
View File
@@ -1,4 +1,5 @@
// Cron stagger tests cover deterministic schedule spreading across jobs.
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it } from "vitest";
import { normalizeCronStaggerMs, resolveCronStaggerMs } from "./stagger.js";
@@ -40,6 +41,7 @@ describe("cron stagger helpers", () => {
expect(normalizeCronStaggerMs("abc")).toBeUndefined();
expect(normalizeCronStaggerMs("1e3")).toBeUndefined();
expect(normalizeCronStaggerMs("0x10")).toBeUndefined();
expect(normalizeCronStaggerMs(MAX_DATE_TIMESTAMP_MS + 1)).toBeUndefined();
expect(normalizeCronStaggerMs(Number.MAX_SAFE_INTEGER + 1)).toBeUndefined();
});
+5 -1
View File
@@ -1,4 +1,8 @@
import { expectDefined } from "@openclaw/normalization-core";
import {
asSafeIntegerInRange,
MAX_DATE_TIMESTAMP_MS,
} from "@openclaw/normalization-core/number-coercion";
/** Resolves deterministic cron stagger windows for recurring schedules. */
import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js";
import type { CronSchedule } from "./types.js";
@@ -53,7 +57,7 @@ export function normalizeCronStaggerMs(raw: unknown): number | undefined {
return undefined;
}
const normalized = Math.max(0, Math.floor(numeric));
return Number.isSafeInteger(normalized) ? normalized : undefined;
return asSafeIntegerInRange(normalized, { max: MAX_DATE_TIMESTAMP_MS });
}
/** Returns the default anti-thundering-herd stagger for top-of-hour recurring schedules. */
+35 -36
View File
@@ -110,28 +110,27 @@ function mergeFailureDestinationProjection(
}
// Empty SQLite sentinels preserve explicit undefined fields for failure
// destination overrides; project them back into the config sidecar shape.
const delivery: Record<string, unknown> =
isRecord(configJob.delivery) && !Array.isArray(configJob.delivery)
? { ...configJob.delivery }
: projectedJob?.delivery
? {
mode: projectedJob.delivery.mode,
...(projectedJob.delivery.channel ? { channel: projectedJob.delivery.channel } : {}),
...(projectedJob.delivery.to ? { to: projectedJob.delivery.to } : {}),
...(projectedJob.delivery.threadId !== undefined
? { threadId: projectedJob.delivery.threadId }
: {}),
...(projectedJob.delivery.accountId
? { accountId: projectedJob.delivery.accountId }
: {}),
...(projectedJob.delivery.bestEffort !== undefined
? { bestEffort: projectedJob.delivery.bestEffort }
: {}),
...(projectedJob.delivery.completionDestination
? { completionDestination: projectedJob.delivery.completionDestination }
: {}),
}
: {};
const delivery: Record<string, unknown> = isRecord(configJob.delivery)
? { ...configJob.delivery }
: projectedJob?.delivery
? {
mode: projectedJob.delivery.mode,
...(projectedJob.delivery.channel ? { channel: projectedJob.delivery.channel } : {}),
...(projectedJob.delivery.to ? { to: projectedJob.delivery.to } : {}),
...(projectedJob.delivery.threadId !== undefined
? { threadId: projectedJob.delivery.threadId }
: {}),
...(projectedJob.delivery.accountId
? { accountId: projectedJob.delivery.accountId }
: {}),
...(projectedJob.delivery.bestEffort !== undefined
? { bestEffort: projectedJob.delivery.bestEffort }
: {}),
...(projectedJob.delivery.completionDestination
? { completionDestination: projectedJob.delivery.completionDestination }
: {}),
}
: {};
const nextFailureDestination = isRecord(delivery.failureDestination)
? { ...delivery.failureDestination }
: {};
@@ -227,7 +226,7 @@ export function assertCronStoreCanPersist(store: CronStoreFile): void {
}
}
function scheduleFromRow(row: CronJobRow): CronSchedule | null {
function scheduleFromRow(row: CronJobRow, jobJson: Record<string, unknown>): CronSchedule | null {
if (row.schedule_kind === "at" && row.at) {
return { kind: "at", at: row.at };
}
@@ -254,7 +253,7 @@ function scheduleFromRow(row: CronJobRow): CronSchedule | null {
};
}
if (row.schedule_kind === "stream") {
const schedule = asOptionalObjectRecord(safeParseJson(row.job_json))?.schedule;
const schedule = jobJson.schedule;
if (!isRecord(schedule) || schedule.kind !== "stream" || !Array.isArray(schedule.command)) {
return null;
}
@@ -263,9 +262,9 @@ function scheduleFromRow(row: CronJobRow): CronSchedule | null {
return null;
}
function pacingFromRow(row: CronJobRow): CronPacing | undefined {
const pacing = asOptionalObjectRecord(safeParseJson(row.job_json))?.pacing;
if (!isRecord(pacing) || Array.isArray(pacing)) {
function pacingFromJobJson(jobJson: Record<string, unknown>): CronPacing | undefined {
const pacing = jobJson.pacing;
if (!isRecord(pacing)) {
return undefined;
}
return {
@@ -274,18 +273,17 @@ function pacingFromRow(row: CronJobRow): CronPacing | undefined {
};
}
function rowToCronJob(row: CronJobRow): CronStoredJob | null {
const jobJson = asOptionalObjectRecord(safeParseJson(row.job_json)) ?? {};
function rowToCronJob(row: CronJobRow, jobJson: Record<string, unknown>): CronStoredJob | null {
const jsonOwner = isRecord(jobJson.owner) ? jobJson.owner : undefined;
const ownerAccountId = normalizeOptionalAccountId(
typeof jsonOwner?.accountId === "string" ? jsonOwner.accountId : undefined,
);
const schedule = scheduleFromRow(row);
const schedule = scheduleFromRow(row, jobJson);
const payload = payloadFromRow(row);
const delivery = deliveryFromRow(row);
const failureAlert = failureAlertFromRow(row);
const trigger = triggerFromRow(row);
const pacing = pacingFromRow(row);
const pacing = pacingFromJobJson(jobJson);
const scheduledToolPolicy = normalizeCronScheduledToolPolicy(jobJson.scheduledToolPolicy);
const toolsAllowProvenance =
isRecord(jobJson.toolsAllowProvenance) &&
@@ -342,7 +340,7 @@ export function projectCronJobThroughStorageCodec(job: CronStoredJob): CronStore
throw new Error(`cannot project invalid cron job ${job.id}`);
}
const row = bindCronJobRow("config-revision", normalized, 0) as CronJobRow;
const projected = rowToCronJob(row);
const projected = rowToCronJob(row, asOptionalObjectRecord(safeParseJson(row.job_json)) ?? {});
if (!projected) {
throw new Error(`cannot project cron job ${job.id} through storage codecs`);
}
@@ -487,10 +485,11 @@ export function loadedCronStoreFromRows(rows: CronJobRow[]): LoadedCronStore {
const invalidConfigRows: LoadedCronStore["invalidConfigRows"] = [];
for (const [index, row] of rows.entries()) {
const job = rowToCronJob(row);
const parsedJobJson = asOptionalObjectRecord(safeParseJson(row.job_json));
const jobJson = parsedJobJson ?? {};
const job = rowToCronJob(row, jobJson);
const configJob = mergeFailureDestinationProjection(
asOptionalObjectRecord(safeParseJson(row.job_json)) ??
(job ? stripJobRuntimeFields(job) : {}),
parsedJobJson ?? (job ? stripJobRuntimeFields(job) : {}),
job,
);
const runtimeEntry = {
@@ -504,7 +503,7 @@ export function loadedCronStoreFromRows(rows: CronJobRow[]): LoadedCronStore {
sourceIndex: index,
reason:
getInvalidPersistedCronJobReason(configJob) ??
(scheduleFromRow(row) ? "invalid-payload" : "invalid-schedule"),
(scheduleFromRow(row, jobJson) ? "invalid-payload" : "invalid-schedule"),
job: configJob,
...(runtimeEntry.state ? { state: runtimeEntry.state } : {}),
...(runtimeEntry.updatedAtMs !== undefined
+43 -38
View File
@@ -1,6 +1,10 @@
/** Read-side cron codec between task-ledger detail and the stable run-history wire shape.
* Deliberately free of agent/runtime imports so history reads stay dependency-light;
* the event->entry write codec lives in task-run-event-codec.ts. */
import {
asSafeIntegerInRange,
MAX_DATE_TIMESTAMP_MS,
} from "@openclaw/normalization-core/number-coercion";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import {
FAILOVER_REASONS,
@@ -24,14 +28,38 @@ function toJsonValue(value: unknown): JsonValue | undefined {
return serialized === undefined ? undefined : (JSON.parse(serialized) as JsonValue);
}
function isJsonObject(value: JsonValue | undefined): value is { [key: string]: JsonValue } {
function isJsonObject(value: unknown): value is { [key: string]: JsonValue } {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function isCronRunStatus(value: unknown): value is CronRunStatus {
function normalizeTimestamp(value: unknown): number | undefined {
return asSafeIntegerInRange(value, { min: 0, max: MAX_DATE_TIMESTAMP_MS });
}
export function isCronRunStatus(value: unknown): value is CronRunStatus {
return value === "ok" || value === "error" || value === "skipped";
}
export function isCronDeliveryStatus(value: unknown): value is CronDeliveryStatus {
return ["delivered", "not-delivered", "unknown", "not-requested"].includes(
value as CronDeliveryStatus,
);
}
function normalizeUsage(value: unknown): CronRunLogEntry["usage"] {
if (!isJsonObject(value)) {
return undefined;
}
const usage = {
input_tokens: asSafeIntegerInRange(value.input_tokens, { min: 0 }),
output_tokens: asSafeIntegerInRange(value.output_tokens, { min: 0 }),
total_tokens: asSafeIntegerInRange(value.total_tokens, { min: 0 }),
cache_read_tokens: asSafeIntegerInRange(value.cache_read_tokens, { min: 0 }),
cache_write_tokens: asSafeIntegerInRange(value.cache_write_tokens, { min: 0 }),
};
return Object.values(usage).some((tokenCount) => tokenCount !== undefined) ? usage : undefined;
}
function normalizeCronRunLogErrorReason(value: unknown): FailoverReason | undefined {
return typeof value === "string" && CRON_FAILOVER_REASONS.has(value as FailoverReason)
? (value as FailoverReason)
@@ -54,17 +82,14 @@ export function parseCronRunLogEntryObject(
if (typeof entryObj.jobId !== "string" || entryObj.jobId.trim().length === 0) {
return null;
}
if (typeof entryObj.ts !== "number" || !Number.isFinite(entryObj.ts)) {
const ts = normalizeTimestamp(entryObj.ts);
if (ts === undefined) {
return null;
}
if (jobId && entryObj.jobId !== jobId) {
return null;
}
const usage =
entryObj.usage && typeof entryObj.usage === "object"
? (entryObj.usage as Record<string, unknown>)
: undefined;
const normalizedError = typeof entryObj.error === "string" ? entryObj.error : undefined;
const normalizedProvider =
typeof entryObj.provider === "string" && entryObj.provider.trim()
@@ -72,43 +97,28 @@ export function parseCronRunLogEntryObject(
: undefined;
// Diagnostics are redacted at authoring; this read/migration path only normalizes stored shape.
const entry: CronRunLogEntry = {
ts: entryObj.ts,
ts,
jobId: entryObj.jobId,
action: "finished",
status: entryObj.status,
status: isCronRunStatus(entryObj.status) ? entryObj.status : undefined,
error: normalizedError,
errorReason: normalizeCronRunLogErrorReason(entryObj.errorReason) ?? undefined,
summary: entryObj.summary,
summary: typeof entryObj.summary === "string" ? entryObj.summary : undefined,
runId: typeof entryObj.runId === "string" && entryObj.runId.trim() ? entryObj.runId : undefined,
diagnostics: normalizeCronRunDiagnostics(entryObj.diagnostics),
runAtMs: entryObj.runAtMs,
durationMs: entryObj.durationMs,
nextRunAtMs: entryObj.nextRunAtMs,
runAtMs: normalizeTimestamp(entryObj.runAtMs),
durationMs: asSafeIntegerInRange(entryObj.durationMs, { min: 0 }),
nextRunAtMs: normalizeTimestamp(entryObj.nextRunAtMs),
triggerFired: entryObj.triggerFired === true ? true : undefined,
model: typeof entryObj.model === "string" && entryObj.model.trim() ? entryObj.model : undefined,
provider: normalizedProvider,
usage: usage
? {
input_tokens: typeof usage.input_tokens === "number" ? usage.input_tokens : undefined,
output_tokens: typeof usage.output_tokens === "number" ? usage.output_tokens : undefined,
total_tokens: typeof usage.total_tokens === "number" ? usage.total_tokens : undefined,
cache_read_tokens:
typeof usage.cache_read_tokens === "number" ? usage.cache_read_tokens : undefined,
cache_write_tokens:
typeof usage.cache_write_tokens === "number" ? usage.cache_write_tokens : undefined,
}
: undefined,
usage: normalizeUsage(entryObj.usage),
};
if (typeof entryObj.delivered === "boolean") {
entry.delivered = entryObj.delivered;
}
if (
entryObj.deliveryStatus === "delivered" ||
entryObj.deliveryStatus === "not-delivered" ||
entryObj.deliveryStatus === "unknown" ||
entryObj.deliveryStatus === "not-requested"
) {
entry.deliveryStatus = entryObj.deliveryStatus as CronDeliveryStatus;
if (isCronDeliveryStatus(entryObj.deliveryStatus)) {
entry.deliveryStatus = entryObj.deliveryStatus;
}
if (typeof entryObj.deliveryError === "string") {
entry.deliveryError = entryObj.deliveryError;
@@ -122,12 +132,7 @@ export function parseCronRunLogEntryObject(
status?: unknown;
error?: unknown;
};
if (
failureNotificationDelivery.status === "delivered" ||
failureNotificationDelivery.status === "not-delivered" ||
failureNotificationDelivery.status === "unknown" ||
failureNotificationDelivery.status === "not-requested"
) {
if (isCronDeliveryStatus(failureNotificationDelivery.status)) {
entry.failureNotificationDelivery = {
status: failureNotificationDelivery.status,
...(typeof failureNotificationDelivery.delivered === "boolean"
@@ -139,7 +144,7 @@ export function parseCronRunLogEntryObject(
};
}
}
if (entryObj.delivery && typeof entryObj.delivery === "object") {
if (isJsonObject(entryObj.delivery)) {
entry.delivery = entryObj.delivery;
}
if (typeof entryObj.sessionId === "string" && entryObj.sessionId.trim()) {
+54 -1
View File
@@ -1,4 +1,5 @@
import { expectDefined } from "@openclaw/normalization-core";
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
import { describe, expect, it, vi } from "vitest";
import { FAILOVER_REASONS } from "../../packages/gateway-protocol/src/failover-reasons.js";
import { saveTaskRegistryStateToSqlite } from "../tasks/task-registry.store.sqlite.js";
@@ -441,16 +442,27 @@ describe("cron task run history", () => {
);
});
it("keeps the internal store key out of the legacy wire record", () => {
it("allowlists the legacy wire record", () => {
const storeKey = "/internal/cron/store";
const task = taskFromEntry(
{ ts: 100, jobId: JOB_ID, action: "finished", status: "ok" },
1,
storeKey,
);
task.detail = {
...(task.detail as Record<string, TaskRecord["detail"]>),
internalFutureField: "secret",
triggerState: { secret: true },
delivery: "malformed",
failureNotificationDelivery: { status: "invalid", internal: "secret" },
};
const entry = cronTaskRecordToRunLogEntry(task);
expect(entry).not.toBeNull();
expect(Object.hasOwn(entry ?? {}, "storeKey")).toBe(false);
expect(Object.hasOwn(entry ?? {}, "internalFutureField")).toBe(false);
expect(Object.hasOwn(entry ?? {}, "triggerState")).toBe(false);
expect(entry?.delivery).toBeUndefined();
expect(entry?.failureNotificationDelivery).toBeUndefined();
});
it("locks the serialized detail shape: kind first, status second", () => {
@@ -494,6 +506,47 @@ describe("cron task run history", () => {
).toBeUndefined();
});
it("rejects invalid legacy run-history scalar and timestamp fields", () => {
const base = { ts: 100, jobId: JOB_ID, action: "finished" } as const;
expect(
parseCronRunLogEntryObject({
...base,
status: "invalid",
summary: 42,
runAtMs: -1,
durationMs: 1.5,
nextRunAtMs: MAX_DATE_TIMESTAMP_MS + 1,
delivery: [],
usage: { input_tokens: Number.NaN, output_tokens: -1 },
}),
).toEqual({
...base,
status: undefined,
error: undefined,
errorReason: undefined,
summary: undefined,
runId: undefined,
diagnostics: undefined,
runAtMs: undefined,
durationMs: undefined,
nextRunAtMs: undefined,
triggerFired: undefined,
model: undefined,
provider: undefined,
usage: undefined,
});
expect(parseCronRunLogEntryObject({ ...base, usage: [] })?.usage).toBeUndefined();
expect(parseCronRunLogEntryObject({ ...base, usage: { input_tokens: 0 } })?.usage).toEqual({
input_tokens: 0,
output_tokens: undefined,
total_tokens: undefined,
cache_read_tokens: undefined,
cache_write_tokens: undefined,
});
expect(parseCronRunLogEntryObject({ ...base, ts: MAX_DATE_TIMESTAMP_MS })).not.toBeNull();
expect(parseCronRunLogEntryObject({ ...base, ts: MAX_DATE_TIMESTAMP_MS + 1 })).toBeNull();
});
it("preserves every canonical failover reason in stored run history", () => {
for (const errorReason of FAILOVER_REASONS) {
const entry = {
+6 -14
View File
@@ -7,7 +7,12 @@ import { uniqueValues } from "@openclaw/normalization-core/string-normalization"
import { listTaskRegistryRecordsByRuntimeSourceIdFromSqlite } from "../tasks/task-registry.store.sqlite.js";
import type { TaskRecord } from "../tasks/task-registry.types.js";
import type { CronRunLogEntry } from "./run-log-types.js";
import { cronTaskRecordStoreKey, cronTaskRecordToRunLogEntry } from "./task-run-detail.js";
import {
cronTaskRecordStoreKey,
cronTaskRecordToRunLogEntry,
isCronDeliveryStatus,
isCronRunStatus,
} from "./task-run-detail.js";
import type { CronDeliveryStatus, CronRunStatus } from "./types.js";
type CronRunHistorySortDir = "asc" | "desc";
@@ -63,19 +68,6 @@ function normalizeStatuses(options: ReadCronTaskRunHistoryPageOptions): CronRunS
return isCronRunStatus(options.status) ? [options.status] : null;
}
function isCronRunStatus(value: unknown): value is CronRunStatus {
return value === "ok" || value === "error" || value === "skipped";
}
function isCronDeliveryStatus(value: unknown): value is CronDeliveryStatus {
return (
value === "delivered" ||
value === "not-delivered" ||
value === "unknown" ||
value === "not-requested"
);
}
function normalizeDeliveryStatuses(
options: ReadCronTaskRunHistoryPageOptions,
): CronDeliveryStatus[] | null {
+1 -1
View File
@@ -1,5 +1,5 @@
import { toErrorObject } from "@openclaw/normalization-core/error-coercion";
import { errorBackoffMs } from "../cron/service/jobs.js";
import { errorBackoffMs } from "../cron/service/jobs-scheduling.js";
import { cronStreamScheduleKey } from "../cron/stream-schedule.js";
import type { CronJob, CronJobState } from "../cron/types.js";
import { formatErrorMessage } from "../infra/errors.js";
+54 -93
View File
@@ -40,11 +40,6 @@ type CronAgentResolver = (requested?: string | null) => {
cfg: OpenClawConfig;
};
type CronWebhookTarget = {
url: string;
source: "completionDestination";
};
type CronFailureAlertParams = {
deps: CliDeps;
logger: CronLogger;
@@ -130,27 +125,20 @@ function redactCommandCronEventForExternalDelivery(evt: CronEvent, job?: CronJob
return redacted;
}
/** Resolves detached completion-destination webhooks. */
function resolveCronWebhookTargets(params: {
function resolveCronCompletionWebhook(params: {
delivery?: {
mode?: string;
to?: string;
completionDestination?: { mode?: string; to?: string };
};
}): CronWebhookTarget[] {
const targets: CronWebhookTarget[] = [];
const mode = normalizeOptionalLowercaseString(params.delivery?.mode);
const completionMode = normalizeOptionalLowercaseString(
params.delivery?.completionDestination?.mode,
);
if (mode === "announce" && completionMode === "webhook") {
const url = normalizeHttpWebhookUrl(params.delivery?.completionDestination?.to);
if (url) {
targets.push({ url, source: "completionDestination" });
}
}): string | undefined {
if (
normalizeOptionalLowercaseString(params.delivery?.mode) !== "announce" ||
normalizeOptionalLowercaseString(params.delivery?.completionDestination?.mode) !== "webhook"
) {
return undefined;
}
return targets;
return normalizeHttpWebhookUrl(params.delivery?.completionDestination?.to) ?? undefined;
}
function buildCronWebhookHeaders(webhookToken?: string): Record<string, string> {
@@ -460,7 +448,7 @@ export function dispatchGatewayCronFinishedNotifications(params: {
params.job?.payload.kind === "script"
? normalizeOptionalString(redactedWebhookEvent.summary)
: params.evt.summary;
const webhookTargets = resolveCronWebhookTargets({
const completionWebhookUrl = resolveCronCompletionWebhook({
delivery:
params.job?.delivery && typeof params.job.delivery.mode === "string"
? {
@@ -486,27 +474,23 @@ export function dispatchGatewayCronFinishedNotifications(params: {
// Script notify is carried as the completion summary, so its absence uses
// the same silent-summary suppression path as NO_REPLY output.
if (completionSummary || params.evt.status === "error") {
for (const webhookTarget of webhookTargets) {
const payload = buildCronFinishedWebhookPayload(redactedWebhookEvent);
// Completion notification fanout is best-effort; the cron service has
// already recorded the run result and must not wait on slow webhooks.
dispatchDetachedCronNotification({
jobId: params.evt.jobId,
logger: params.logger,
deliver: () =>
postCronWebhook({
webhookUrl: webhookTarget.url,
webhookToken,
ssrfPolicy: params.ssrfPolicy,
payload,
logContext: { jobId: params.evt.jobId, source: webhookTarget.source },
blockedLog: "cron: webhook delivery blocked by SSRF guard",
failedLog: "cron: webhook delivery failed",
logger: params.logger,
}),
});
}
if (completionWebhookUrl && (completionSummary || params.evt.status === "error")) {
const payload = buildCronFinishedWebhookPayload(redactedWebhookEvent);
dispatchDetachedCronNotification({
jobId: params.evt.jobId,
logger: params.logger,
deliver: () =>
postCronWebhook({
webhookUrl: completionWebhookUrl,
webhookToken,
ssrfPolicy: params.ssrfPolicy,
payload,
logContext: { jobId: params.evt.jobId, source: "completionDestination" },
blockedLog: "cron: webhook delivery blocked by SSRF guard",
failedLog: "cron: webhook delivery failed",
logger: params.logger,
}),
});
}
dispatchCronFailureDestinationNotifications({
@@ -573,41 +557,31 @@ function dispatchCronFailureDestinationNotifications(params: {
return;
}
if (failureDest.mode === "announce") {
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(job.agentId);
dispatchDetachedCronNotification({
jobId: job.id,
logger: params.logger,
deliver: () =>
sendFailureNotificationAnnounce(
params.deps,
runtimeConfig,
agentId,
job.id,
{
channel: failureDest.channel,
to: failureDest.to,
accountId: failureDest.accountId,
sessionKey: deliverySessionKey,
// A configured failure route is already explicit; keep the cron run
// session only for context, not for reattaching the primary topic.
inheritSessionThread: false,
},
{
text: appendCronRunStarted(
`⚠️ ${failurePayload.message}`,
params.evt.runAtMs,
runtimeConfig,
),
},
),
});
if (failureDest.mode !== "announce") {
return;
}
return;
}
const primaryPlan = resolveCronDeliveryPlan(job);
if (primaryPlan.mode !== "announce" || !primaryPlan.requested) {
const announceTarget = failureDest
? {
channel: failureDest.channel,
to: failureDest.to,
accountId: failureDest.accountId,
sessionKey: deliverySessionKey,
// Explicit failure routes keep run context without inheriting the primary topic.
inheritSessionThread: false,
}
: primaryPlan.mode === "announce" && primaryPlan.requested
? {
channel: primaryPlan.channel,
to: primaryPlan.to,
accountId: primaryPlan.accountId,
threadId: primaryPlan.threadId,
sessionKey: deliverySessionKey,
}
: undefined;
if (!announceTarget) {
return;
}
@@ -616,25 +590,12 @@ function dispatchCronFailureDestinationNotifications(params: {
jobId: job.id,
logger: params.logger,
deliver: () =>
sendFailureNotificationAnnounce(
params.deps,
runtimeConfig,
agentId,
job.id,
{
channel: primaryPlan.channel,
to: primaryPlan.to,
accountId: primaryPlan.accountId,
threadId: primaryPlan.threadId,
sessionKey: deliverySessionKey,
},
{
text: appendCronRunStarted(
`⚠️ ${failurePayload.message}`,
params.evt.runAtMs,
runtimeConfig,
),
},
),
sendFailureNotificationAnnounce(params.deps, runtimeConfig, agentId, job.id, announceTarget, {
text: appendCronRunStarted(
`⚠️ ${failurePayload.message}`,
params.evt.runAtMs,
runtimeConfig,
),
}),
});
}
+119 -132
View File
@@ -222,6 +222,86 @@ function sanitizeCronHeartbeatOverride(
return heartbeat?.target === "last" ? omitExplicitHeartbeatDestination(heartbeat) : heartbeat;
}
async function finalizeCronCompletionAnnouncement(params: {
job: CronJob;
text?: string;
abortSignal?: AbortSignal;
deps: CliDeps;
resolveCronAgent: (requested?: string | null) => { agentId: string; cfg: OpenClawConfig };
logger: ReturnType<typeof getChildLogger>;
label: string;
traceResolvedFailure?: boolean;
}) {
const plan = resolveCronDeliveryPlan(params.job);
const delivery = {
intended: pickDefined(
{
channel: plan.channel,
to: plan.to,
accountId: plan.accountId,
threadId: plan.threadId,
source: "explicit" as const,
},
["channel", "to", "accountId", "threadId", "source"],
),
};
if (plan.mode !== "announce" || params.text === undefined) {
return { deliveryAttempted: false, delivered: false, delivery };
}
const { agentId, cfg } = params.resolveCronAgent(params.job.agentId);
try {
await sendCronAnnouncePayloadStrict({
deps: params.deps,
cfg,
agentId,
jobId: params.job.id,
target: {
channel: plan.channel,
to: plan.to,
threadId: plan.threadId,
accountId: plan.accountId,
sessionKey: resolveCronDeliverySessionKey(params.job),
},
payload: { text: params.text },
abortSignal: params.abortSignal ?? new AbortController().signal,
});
return {
deliveryAttempted: true,
delivered: true,
delivery: { ...delivery, delivered: true },
};
} catch (err) {
const deliveryError = formatErrorMessage(err);
params.logger.warn(
{ jobId: params.job.id, err: deliveryError },
`cron: ${params.label} delivery failed`,
);
return {
deliveryAttempted: true,
delivered: false,
deliveryError,
delivery: {
...delivery,
delivered: false,
...(params.traceResolvedFailure
? {
resolved: {
channel: plan.channel,
to: plan.to,
accountId: plan.accountId,
threadId: plan.threadId,
source: "explicit" as const,
ok: false,
error: deliveryError,
},
}
: {}),
},
};
}
}
/** Map internal CronJob to the public plugin SDK shape. */
function toPluginCronJob(job: CronJob): PluginHookGatewayCronJob {
return {
@@ -746,104 +826,45 @@ export function buildGatewayCronService(params: {
abortSignal,
nowMs: Date.now,
});
const plan = resolveCronDeliveryPlan(job);
const deliveryTrace = {
intended: pickDefined(
{
channel: plan.channel,
to: plan.to,
threadId: plan.threadId,
accountId: plan.accountId,
source: "explicit" as const,
},
["channel", "to", "accountId", "threadId", "source"],
),
};
const summaryIsSilent =
typeof result.summary === "string" && isSilentReplyText(result.summary, SILENT_REPLY_TOKEN);
if (summaryIsSilent) {
const { summary: _summary, ...silentResult } = result;
return {
...silentResult,
deliveryAttempted: false,
delivered: false,
delivery: deliveryTrace,
};
}
const shouldAnnounce =
plan.mode === "announce" && typeof result.summary === "string" && result.summary.trim();
if (!shouldAnnounce) {
return {
...result,
deliveryAttempted: false,
delivered: false,
delivery: deliveryTrace,
};
}
const message = isCommandCronJob(job)
? redactCronCommandSummaryForExternalDelivery(result.summary)
: result.summary;
if (typeof message !== "string") {
return {
...result,
deliveryAttempted: false,
delivered: false,
delivery: deliveryTrace,
};
}
const { agentId, cfg: runtimeConfig } = resolveCronAgent(job.agentId);
try {
await sendCronAnnouncePayloadStrict({
const completion = await finalizeCronCompletionAnnouncement({
job,
deps: params.deps,
cfg: runtimeConfig,
agentId,
jobId: job.id,
target: {
channel: plan.channel,
to: plan.to,
threadId: plan.threadId,
accountId: plan.accountId,
sessionKey: resolveCronDeliverySessionKey(job),
},
payload: { text: message },
abortSignal: abortSignal ?? new AbortController().signal,
resolveCronAgent,
logger: cronLogger,
label: "command",
});
return {
...result,
deliveryAttempted: true,
delivered: true,
delivery: {
...deliveryTrace,
delivered: true,
},
};
} catch (err) {
const error = formatErrorMessage(err);
return { ...silentResult, ...completion };
}
const completion = await finalizeCronCompletionAnnouncement({
job,
text:
typeof result.summary === "string" && result.summary.trim()
? redactCronCommandSummaryForExternalDelivery(result.summary)
: undefined,
abortSignal,
deps: params.deps,
resolveCronAgent,
logger: cronLogger,
label: "command",
traceResolvedFailure: true,
});
if ("deliveryError" in completion) {
const { deliveryError, ...deliveryResult } = completion;
const requiredDeliveryFailed = job.delivery?.bestEffort === false && result.status === "ok";
cronLogger.warn({ jobId: job.id, err: error }, "cron: command delivery failed");
return {
...result,
// Default announce delivery is best-effort, but an explicit
// bestEffort:false keeps delivery inside the job's success contract.
status: requiredDeliveryFailed ? ("error" as const) : result.status,
...(requiredDeliveryFailed ? { error } : { deliveryError: error }),
deliveryAttempted: true,
delivered: false,
delivery: {
...deliveryTrace,
delivered: false,
resolved: {
channel: plan.channel,
to: plan.to,
accountId: plan.accountId,
threadId: plan.threadId,
source: "explicit" as const,
ok: false,
error,
},
},
...(requiredDeliveryFailed ? { error: deliveryError } : { deliveryError }),
...deliveryResult,
};
}
return { ...result, ...completion };
},
sendCronWebhook: async ({ job, event, abortSignal, deadlineAtMs, onDeliveryAccepted }) => {
await sendGatewayCronWebhook({
@@ -891,19 +912,6 @@ export function buildGatewayCronService(params: {
}
const notify = execution.notify?.trim() ? execution.notify : undefined;
const plan = resolveCronDeliveryPlan(job);
const deliveryTrace = {
intended: pickDefined(
{
channel: plan.channel,
to: plan.to,
accountId: plan.accountId,
threadId: plan.threadId,
source: "explicit" as const,
},
["channel", "to", "accountId", "threadId", "source"],
),
};
const base = {
status: "ok" as const,
notify,
@@ -911,47 +919,26 @@ export function buildGatewayCronService(params: {
stateChanged: execution.stateChanged,
...(execution.stateChanged ? { state: execution.state } : {}),
nextCheck: execution.nextCheck,
delivery: deliveryTrace,
};
if (job.sessionTarget === "main" || plan.mode !== "announce" || !notify) {
return { ...base, deliveryAttempted: false, delivered: false };
}
const { agentId, cfg: runtimeConfig } = resolveCronAgent(job.agentId);
try {
await sendCronAnnouncePayloadStrict({
deps: params.deps,
cfg: runtimeConfig,
agentId,
jobId: job.id,
target: {
channel: plan.channel,
to: plan.to,
threadId: plan.threadId,
accountId: plan.accountId,
sessionKey: resolveCronDeliverySessionKey(job),
},
payload: { text: notify },
abortSignal: abortSignal ?? new AbortController().signal,
});
return {
...base,
deliveryAttempted: true,
delivered: true,
delivery: { ...deliveryTrace, delivered: true },
};
} catch (err) {
const error = formatErrorMessage(err);
cronLogger.warn({ jobId: job.id, err: error }, "cron: script payload delivery failed");
const completion = await finalizeCronCompletionAnnouncement({
job,
text: job.sessionTarget === "main" ? undefined : notify,
abortSignal,
deps: params.deps,
resolveCronAgent,
logger: cronLogger,
label: "script payload",
});
if ("deliveryError" in completion) {
const { deliveryError, ...deliveryResult } = completion;
return {
...base,
status: job.delivery?.bestEffort ? ("ok" as const) : ("error" as const),
...(job.delivery?.bestEffort ? { deliveryError: error } : { error }),
deliveryAttempted: true,
delivered: false,
delivery: { ...deliveryTrace, delivered: false },
...(job.delivery?.bestEffort ? { deliveryError } : { error: deliveryError }),
...deliveryResult,
};
}
return { ...base, ...completion };
},
cleanupTimedOutAgentRun: async ({ job, execution }) => {
if (!execution?.sessionId) {
+4
View File
@@ -8,6 +8,8 @@ describe("hasValidIsoCalendarComponents", () => {
"2028-02-29T12:30:45.123456+01:30",
"2028-02-29T24:00:00Z",
"2028-02-29T24:00:00.0000Z",
"+275760-09-13T00:00:00.000Z",
"-271821-04-20T00:00:00.000Z",
])("accepts valid calendar components in %s", (value) => {
expect(hasValidIsoCalendarComponents(value)).toBe(true);
});
@@ -21,6 +23,8 @@ describe("hasValidIsoCalendarComponents", () => {
"2026-07-05T12:60:00Z",
"2026-07-05T12:00:60Z",
"2026-7-05",
"+275760-09-13T00:00:00.001Z",
"275760-09-13T00:00:00.000Z",
])("rejects invalid calendar components or shape in %s", (value) => {
expect(hasValidIsoCalendarComponents(value)).toBe(false);
});
+1 -1
View File
@@ -1,5 +1,5 @@
const ISO_ABSOLUTE_RE =
/^(\d{4})-(\d{2})-(\d{2})(?:[Tt](\d{2}):(\d{2})(?::(\d{2})(\.\d+)?)?(?:[Zz]|[+-]\d{2}:?\d{2})?)?$/;
/^([+-]\d{6}|\d{4})-(\d{2})-(\d{2})(?:[Tt](\d{2}):(\d{2})(?::(\d{2})(\.\d+)?)?(?:[Zz]|[+-]\d{2}:?\d{2})?)?$/;
/** Checks the calendar components of the ISO-like forms accepted by existing callers. */
export function hasValidIsoCalendarComponents(raw: string): boolean {
@@ -261,7 +261,7 @@
"tools": [
{
"deferLoading": true,
"description": "Gateway scheduler: reminders, delayed self-wakeups, loops, recurring work, event watchers. Never exec sleep/poll as timer.\n\nACTIONS: status | list [includeDisabled,limit?,offset?] (use nextOffset for the next page) | get jobId | add job | update jobId patch | remove jobId | run jobId (runMode \"force\"=now) | runs jobId = history | next_check in:\"30m\" (own paced run only) | wake text mode?:\"now\"|\"next-heartbeat\"(default) nudges a caller-owned lane (sessionKey/agentId to pick another).\n\nADD: {name?,schedule,payload,sessionTarget?,pacing?,trigger?,delivery?,enabled?}. Required: schedule+payload.\n\nSCHEDULE:\n- {kind:\"at\",at:\"ISO-8601\"} one-shot; no tz=UTC; auto-deletes after run.\n- {kind:\"every\",everyMs}.\n- {kind:\"cron\",expr,tz?:\"IANA\"}: expr is wall time in tz; never pre-convert to UTC; no tz=gateway host local. 18:00 Shanghai => {expr:\"0 18 * * *\",tz:\"Asia/Shanghai\"}.\n- {kind:\"stream\",command:[argv],mode?:\"line\"|\"match\",match?}: fires on supervised process output; needs cron.triggers.enabled.\n\nTARGET+PAYLOAD:\n- \"current\" (agentTurn default) = this conversation: run carries this chat's context, result lands here. Self-wakeup/\"continue later\"/loop = at|every + agentTurn + current.\n- \"isolated\" = fresh detached session (shows in `openclaw tasks`); standalone background work.\n- \"main\" = heartbeat lane; payload {kind:\"systemEvent\",text} (systemEvent default target).\n- \"session:<key>\" = named session.\n- agentTurn {kind:\"agentTurn\",message,model?,thinking?,timeoutSeconds?}; timeoutSeconds 0=none.\n- Inherited configured MCP authority includes only model-callable tools; interactive app-view-only capabilities are excluded from headless jobs.\n- script {kind:\"script\",script,timeoutSeconds?,toolBudget?}: main|isolated only; needs cron.triggers.enabled.\n\nPACED LOOP: recurring job + pacing{min?,max?} durations (\"15m\",\"4h\"; at least one). Inside its run, job calls next_check in:\"<dur>\" to set the next delay (clamped to bounds, measured from run end; failed runs keep normal backoff). Adaptive polling: tighten when active, back off when quiet.\n\nTRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call(\"exec\",{command:\"...\"}).\n\nDELIVERY {mode:\"none\"|\"announce\"|\"webhook\",channel?,to?,threadId?,bestEffort?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:\"none\". webhook posts finished-run event to URL in `to`.\n\nJob wakeMode (main jobs): \"now\"(default)|\"next-heartbeat\". Restricted automation-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.",
"description": "Gateway scheduler: reminders, delayed self-wakeups, loops, recurring work, event watchers. Never exec sleep/poll as timer.\n\nACTIONS: status | list [includeDisabled,limit?,offset?] (use nextOffset for the next page) | get jobId | add job | update jobId patch | remove jobId | run jobId (runMode \"force\"=now) | runs jobId = history | next_check in:\"30m\" (own paced run only) | wake text mode?:\"now\"|\"next-heartbeat\"(default) nudges a caller-owned lane (sessionKey/agentId to pick another).\n\nADD: {name?,schedule,payload,sessionTarget?,pacing?,trigger?,delivery?,enabled?}. Required: schedule+payload.\n\nSCHEDULE:\n- {kind:\"at\",at:\"ISO-8601\"} one-shot; no tz=UTC; auto-deletes after run.\n- {kind:\"every\",everyMs}.\n- {kind:\"cron\",expr,tz?:\"IANA\"}: expr is wall time in tz; never pre-convert to UTC; no tz=gateway host local. 18:00 Shanghai => {expr:\"0 18 * * *\",tz:\"Asia/Shanghai\"}.\n- {kind:\"stream\",command:[argv],mode?:\"line\"|\"match\",match?}: fires on supervised process output; needs cron.triggers.enabled.\n\nTARGET+PAYLOAD:\n- \"current\" (agentTurn default) = this conversation: run carries this chat's context, result lands here. Self-wakeup/\"continue later\"/loop = at|every + agentTurn + current.\n- \"isolated\" = fresh detached session (shows in `openclaw tasks`); standalone background work.\n- \"main\" = heartbeat lane; payload {kind:\"systemEvent\",text} (systemEvent default target).\n- \"session:<key>\" = named session.\n- agentTurn {kind:\"agentTurn\",message,model?,thinking?,timeoutSeconds?}; timeoutSeconds 0=none.\n- Inherited configured MCP authority includes only model-callable tools; interactive app-view-only capabilities are excluded from headless jobs.\n- script {kind:\"script\",script,timeoutSeconds?,toolBudget?}: main|isolated only; needs cron.triggers.enabled.\n\nPACED LOOP: recurring job + pacing{min?,max?} durations (\"15m\",\"4h\"; at least one). Inside its run, job calls next_check in:\"<dur>\" to set the next delay (clamped to bounds, measured from run end; failed runs keep normal backoff). Adaptive polling: tighten when active, back off when quiet.\n\nTRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call(\"exec\",{command:\"...\"}).\n\nDELIVERY {mode:\"none\"|\"announce\"|\"webhook\",channel?,to?,threadId?,bestEffort?,completionDestination?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:\"none\". webhook posts finished-run event to URL in `to`. To keep announce delivery and also POST completion, use mode:\"announce\" with completionDestination:{mode:\"webhook\",to:\"https://...\"}.\n\nJob wakeMode (main jobs): \"now\"(default)|\"next-heartbeat\". Restricted automation-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.",
"inputSchema": {
"additionalProperties": true,
"properties": {
@@ -343,6 +343,23 @@
"description": "Delivery channel",
"type": "string"
},
"completionDestination": {
"additionalProperties": true,
"description": "Additional completion webhook; requires delivery.mode=announce",
"properties": {
"mode": {
"const": "webhook",
"type": "string"
},
"to": {
"description": "Completion webhook target; only valid with delivery.mode=announce",
"minLength": 1,
"type": "string"
}
},
"required": ["mode", "to"],
"type": "object"
},
"failureDestination": {
"additionalProperties": true,
"properties": {
@@ -542,6 +559,7 @@
"properties": {
"anchorMs": {
"description": "Start anchor ms (kind=every)",
"maximum": 8640000000000000,
"minimum": 0,
"type": "integer"
},
@@ -568,6 +586,7 @@
},
"everyMs": {
"description": "Interval ms (kind=every)",
"maximum": 8640000000000000,
"minimum": 1,
"type": "integer"
},
@@ -594,6 +613,7 @@
},
"staggerMs": {
"description": "Jitter ms (kind=cron)",
"maximum": 8640000000000000,
"minimum": 0,
"type": "integer"
},
@@ -706,6 +726,31 @@
],
"description": "Delivery channel, or null to clear"
},
"completionDestination": {
"anyOf": [
{
"additionalProperties": true,
"description": "Additional completion webhook; requires delivery.mode=announce",
"properties": {
"mode": {
"const": "webhook",
"type": "string"
},
"to": {
"description": "Completion webhook target; only valid with delivery.mode=announce",
"minLength": 1,
"type": "string"
}
},
"required": ["mode", "to"],
"type": "object"
},
{
"type": "null"
}
],
"description": "Completion webhook destination; requires delivery.mode=announce; null clears."
},
"failureDestination": {
"anyOf": [
{
@@ -969,6 +1014,7 @@
"properties": {
"anchorMs": {
"description": "Start anchor ms (kind=every)",
"maximum": 8640000000000000,
"minimum": 0,
"type": "integer"
},
@@ -995,6 +1041,7 @@
},
"everyMs": {
"description": "Interval ms (kind=every)",
"maximum": 8640000000000000,
"minimum": 1,
"type": "integer"
},
@@ -1021,6 +1068,7 @@
},
"staggerMs": {
"description": "Jitter ms (kind=cron)",
"maximum": 8640000000000000,
"minimum": 0,
"type": "integer"
},
@@ -227,8 +227,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
"roughTokens": 0
},
"dynamicToolsJson": {
"chars": 62876,
"roughTokens": 15719
"chars": 65442,
"roughTokens": 16361
},
"openClawDeveloperInstructions": {
"chars": 4519,
@@ -239,8 +239,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
"roughTokens": 7226
},
"totalWithDynamicToolsJson": {
"chars": 91780,
"roughTokens": 22945
"chars": 94346,
"roughTokens": 23587
},
"userInputText": {
"chars": 1300,
@@ -227,8 +227,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
"roughTokens": 0
},
"dynamicToolsJson": {
"chars": 62568,
"roughTokens": 15642
"chars": 65134,
"roughTokens": 16284
},
"openClawDeveloperInstructions": {
"chars": 3410,
@@ -239,8 +239,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
"roughTokens": 6856
},
"totalWithDynamicToolsJson": {
"chars": 89992,
"roughTokens": 22498
"chars": 92558,
"roughTokens": 23140
},
"userInputText": {
"chars": 929,
@@ -222,8 +222,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
"roughTokens": 0
},
"dynamicToolsJson": {
"chars": 64102,
"roughTokens": 16026
"chars": 66668,
"roughTokens": 16667
},
"openClawDeveloperInstructions": {
"chars": 3410,
@@ -234,8 +234,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
"roughTokens": 6960
},
"totalWithDynamicToolsJson": {
"chars": 91942,
"roughTokens": 22986
"chars": 94508,
"roughTokens": 23627
},
"userInputText": {
"chars": 1271,
+1 -1
View File
@@ -31,7 +31,7 @@ describe("audit-seams cron seam classification", () => {
it("detects scheduler-state seams in cron service orchestration", () => {
const source = `
import { recomputeNextRuns, computeJobNextRunAtMs } from "./jobs.js";
import { recomputeNextRuns, computeJobNextRunAtMs } from "./jobs-scheduling.js";
import { ensureLoaded, persist } from "./store.js";
import { armTimer, runMissedJobs } from "./timer.js";