mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(cron): prevent invalid timestamps from stranding jobs (#121394)
* fix(cron): harden scheduling timestamp boundaries Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe * test(prompts): refresh cron tool snapshots Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe * test(cron): keep config revision fixture Date-valid Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe * refactor(cron): consolidate scheduling lifecycle Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe * refactor(cron): keep task history dependencies acyclic Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe * fix(cron): canonicalize timestamp auto-disable Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe * test(cron): verify startup overflow notifications Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe * chore(ci): repair main baseline gates Amp-Thread-ID: https://ampcode.com/threads/T-019fe949-92e4-76bd-8cfa-aea44fcfaebe --------- Co-authored-by: Amp <amp@ampcode.com>
This commit is contained in:
committed by
GitHub
parent
c6edd3e80f
commit
8e1c238c1c
@@ -1,5 +1,6 @@
|
||||
import { Value } from "typebox/value";
|
||||
// Gateway Protocol tests cover cron validators behavior.
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { Value } from "typebox/value";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
validateCronAddParams,
|
||||
@@ -58,12 +59,21 @@ describe("cron protocol validators", () => {
|
||||
consecutiveErrors: 10,
|
||||
autoDisabled: {
|
||||
reason: "consecutive-failures",
|
||||
atMs: 2,
|
||||
atMs: MAX_DATE_TIMESTAMP_MS,
|
||||
consecutiveErrors: 10,
|
||||
},
|
||||
},
|
||||
};
|
||||
expect(Value.Check(CronJobSchema, job)).toBe(true);
|
||||
expect(
|
||||
Value.Check(CronJobSchema, {
|
||||
...job,
|
||||
state: {
|
||||
...job.state,
|
||||
autoDisabled: { ...job.state.autoDisabled, atMs: MAX_DATE_TIMESTAMP_MS + 1 },
|
||||
},
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(validateCronUpdateParams(update({ state: job.state }))).toBe(false);
|
||||
});
|
||||
|
||||
@@ -100,6 +110,32 @@ describe("cron protocol validators", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects every schedule numbers outside the ECMAScript Date range", () => {
|
||||
const invalidTimestamp = MAX_DATE_TIMESTAMP_MS + 1;
|
||||
expectCases(validateCronAddParams, false, [
|
||||
add({ schedule: { kind: "every", everyMs: invalidTimestamp } }),
|
||||
add({ schedule: { kind: "every", everyMs: 60_000, anchorMs: invalidTimestamp } }),
|
||||
add({ schedule: { kind: "cron", expr: "0 * * * *", staggerMs: invalidTimestamp } }),
|
||||
]);
|
||||
expectCases(validateCronUpdateParams, false, [
|
||||
update({ schedule: { kind: "every", everyMs: invalidTimestamp } }),
|
||||
update({ schedule: { kind: "every", everyMs: 60_000, anchorMs: invalidTimestamp } }),
|
||||
update({ schedule: { kind: "cron", expr: "0 * * * *", staggerMs: invalidTimestamp } }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects mutable scheduler state outside the ECMAScript Date range", () => {
|
||||
const invalidTimestamp = MAX_DATE_TIMESTAMP_MS + 1;
|
||||
expectCases(validateCronUpdateParams, false, [
|
||||
update({ state: { nextRunAtMs: invalidTimestamp } }),
|
||||
update({ state: { runningAtMs: invalidTimestamp } }),
|
||||
update({ state: { lastRunAtMs: invalidTimestamp } }),
|
||||
]);
|
||||
expectCases(validateCronUpdateParams, true, [
|
||||
update({ state: { nextRunAtMs: MAX_DATE_TIMESTAMP_MS } }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("accepts trigger add, patch, and clear shapes", () => {
|
||||
expectCases(validateCronAddParams, true, [
|
||||
add({ trigger: { script: "json({ fire: true })", once: true } }),
|
||||
@@ -235,6 +271,29 @@ describe("cron protocol validators", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("accepts completion webhooks only alongside announce delivery", () => {
|
||||
const completionDestination = {
|
||||
mode: "webhook",
|
||||
to: "https://example.invalid/complete",
|
||||
} as const;
|
||||
expectCases(validateCronAddParams, true, [
|
||||
add({ delivery: { mode: "announce", completionDestination } }),
|
||||
]);
|
||||
expectCases(validateCronAddParams, false, [
|
||||
add({ delivery: { mode: "none", completionDestination } }),
|
||||
add({ delivery: { mode: "webhook", to: "https://example.invalid", completionDestination } }),
|
||||
add({ delivery: { mode: "announce", completionDestination: null } }),
|
||||
]);
|
||||
expectCases(validateCronUpdateParams, true, [
|
||||
update({ delivery: { completionDestination } }),
|
||||
update({ delivery: { completionDestination: null } }),
|
||||
]);
|
||||
expectCases(validateCronUpdateParams, false, [
|
||||
update({ delivery: { completionDestination: {} } }),
|
||||
update({ delivery: { completionDestination: { mode: "announce", to: "https://x.test" } } }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("accepts nullable delivery clears on update params", () => {
|
||||
expectCases(validateCronUpdateParams, true, [
|
||||
update({
|
||||
|
||||
@@ -4,6 +4,14 @@ import { closedObject } from "./closed-object.js";
|
||||
import { FailoverReasonSchema } from "./failover-reason.js";
|
||||
import { NonEmptyString } from "./primitives.js";
|
||||
|
||||
// ECMAScript Date's inclusive timestamp limit. Keep public schedule numbers
|
||||
// inside the range the scheduler can represent and serialize.
|
||||
const MAX_DATE_TIMESTAMP_MS = 8_640_000_000_000_000;
|
||||
const CronDateTimestampMsSchema = Type.Integer({
|
||||
minimum: 0,
|
||||
maximum: MAX_DATE_TIMESTAMP_MS,
|
||||
});
|
||||
|
||||
/**
|
||||
* Cron scheduler protocol schemas.
|
||||
*
|
||||
@@ -205,14 +213,14 @@ const CronScheduleSchema = Type.Union([
|
||||
}),
|
||||
closedObject({
|
||||
kind: Type.Literal("every"),
|
||||
everyMs: Type.Integer({ minimum: 1, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
anchorMs: Type.Optional(Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER })),
|
||||
everyMs: Type.Integer({ minimum: 1, maximum: MAX_DATE_TIMESTAMP_MS }),
|
||||
anchorMs: Type.Optional(Type.Integer({ minimum: 0, maximum: MAX_DATE_TIMESTAMP_MS })),
|
||||
}),
|
||||
closedObject({
|
||||
kind: Type.Literal("cron"),
|
||||
expr: NonEmptyString,
|
||||
tz: Type.Optional(Type.String()),
|
||||
staggerMs: Type.Optional(Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER })),
|
||||
staggerMs: Type.Optional(Type.Integer({ minimum: 0, maximum: MAX_DATE_TIMESTAMP_MS })),
|
||||
}),
|
||||
closedObject({
|
||||
// Event-driven trigger: fires once when the gateway-owned watcher running
|
||||
@@ -415,16 +423,16 @@ const CronFailureNotificationDeliverySchema = closedObject({
|
||||
|
||||
const CronAutoDisabledSchema = closedObject({
|
||||
reason: Type.Union([Type.Literal("consecutive-failures"), Type.Literal("schedule-errors")]),
|
||||
atMs: Type.Integer({ minimum: 0 }),
|
||||
atMs: CronDateTimestampMsSchema,
|
||||
consecutiveErrors: Type.Integer({ minimum: 1 }),
|
||||
});
|
||||
|
||||
/** Scheduler-maintained state for the latest run/delivery outcome. */
|
||||
export const CronJobStateSchema = closedObject({
|
||||
nextRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
scheduleActivatedAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
runningAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
nextRunAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
scheduleActivatedAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
runningAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
lastRunAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
lastRunStatus: Type.Optional(CronRunStatusSchema),
|
||||
lastStatus: Type.Optional(DeprecatedCronRunStatusSchema),
|
||||
lastError: Type.Optional(Type.String()),
|
||||
@@ -442,10 +450,10 @@ export const CronJobStateSchema = closedObject({
|
||||
lastFailureNotificationDelivered: Type.Optional(Type.Boolean()),
|
||||
lastFailureNotificationDeliveryStatus: Type.Optional(CronDeliveryStatusSchema),
|
||||
lastFailureNotificationDeliveryError: Type.Optional(Type.String()),
|
||||
lastFailureAlertAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastTriggerEvalAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastFailureAlertAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
lastTriggerEvalAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
triggerEvalCount: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastTriggerFireAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastTriggerFireAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
triggerState: Type.Optional(Type.Unknown()),
|
||||
streamStatus: Type.Optional(
|
||||
Type.Union([
|
||||
@@ -466,14 +474,14 @@ export const CronJobStateSchema = closedObject({
|
||||
streamSourceIdentity: Type.Optional(Type.String()),
|
||||
streamDroppedBatches: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
streamCoalescedBatches: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
streamLastStartedAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
streamLastExitAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
streamLastStartedAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
streamLastExitAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
});
|
||||
|
||||
const CronJobStatePatchSchema = closedObject({
|
||||
nextRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
runningAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
nextRunAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
runningAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
lastRunAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
lastRunStatus: Type.Optional(CronRunStatusSchema),
|
||||
lastStatus: Type.Optional(DeprecatedCronRunStatusSchema),
|
||||
lastError: Type.Optional(Type.String()),
|
||||
@@ -487,10 +495,10 @@ const CronJobStatePatchSchema = closedObject({
|
||||
lastFailureNotificationDelivered: Type.Optional(Type.Boolean()),
|
||||
lastFailureNotificationDeliveryStatus: Type.Optional(CronDeliveryStatusSchema),
|
||||
lastFailureNotificationDeliveryError: Type.Optional(Type.String()),
|
||||
lastFailureAlertAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastTriggerEvalAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastFailureAlertAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
lastTriggerEvalAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
triggerEvalCount: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastTriggerFireAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastTriggerFireAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
triggerState: Type.Optional(Type.Unknown()),
|
||||
streamStatus: Type.Optional(
|
||||
Type.Union([
|
||||
@@ -507,8 +515,8 @@ const CronJobStatePatchSchema = closedObject({
|
||||
streamRestartExhausted: Type.Optional(Type.Boolean()),
|
||||
streamDroppedBatches: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
streamCoalescedBatches: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
streamLastStartedAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
streamLastExitAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
streamLastStartedAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
streamLastExitAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
});
|
||||
|
||||
/** Persisted cron job definition returned by scheduler list/get APIs. */
|
||||
@@ -524,8 +532,8 @@ export const CronJobSchema = closedObject({
|
||||
description: Type.Optional(Type.String()),
|
||||
enabled: Type.Boolean(),
|
||||
deleteAfterRun: Type.Optional(Type.Boolean()),
|
||||
createdAtMs: Type.Integer({ minimum: 0 }),
|
||||
updatedAtMs: Type.Integer({ minimum: 0 }),
|
||||
createdAtMs: CronDateTimestampMsSchema,
|
||||
updatedAtMs: CronDateTimestampMsSchema,
|
||||
/** Opaque Gateway-computed token for the job definition, excluding scheduler state. */
|
||||
configRevision: Type.Optional(CronConfigRevisionSchema),
|
||||
schedule: CronScheduleSchema,
|
||||
@@ -537,8 +545,8 @@ export const CronJobSchema = closedObject({
|
||||
delivery: Type.Optional(CronDeliverySchema),
|
||||
failureAlert: Type.Optional(Type.Union([Type.Literal(false), CronFailureAlertSchema])),
|
||||
state: CronJobStateSchema,
|
||||
nextRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
lastRunAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
|
||||
nextRunAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
lastRunAtMs: Type.Optional(CronDateTimestampMsSchema),
|
||||
lastRunStatus: Type.Optional(CronRunStatusSchema),
|
||||
lastRunError: Type.Optional(Type.String()),
|
||||
lastDelivered: Type.Optional(Type.Boolean()),
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
/** Model-facing schema and input validation for the cron tool. */
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { Type, type TSchema } from "typebox";
|
||||
import { parseCronPacingBounds } from "../../cron/pacing.js";
|
||||
import type { CronPacing } from "../../cron/types.js";
|
||||
@@ -100,9 +101,13 @@ function createCronScheduleSchema(): TSchema {
|
||||
{
|
||||
kind: optionalStringEnum(CRON_SCHEDULE_KINDS, { description: "Schedule kind" }),
|
||||
at: Type.Optional(Type.String({ description: "ISO-8601 time (kind=at)" })),
|
||||
everyMs: optionalPositiveIntegerSchema({ description: "Interval ms (kind=every)" }),
|
||||
everyMs: optionalPositiveIntegerSchema({
|
||||
description: "Interval ms (kind=every)",
|
||||
maximum: MAX_DATE_TIMESTAMP_MS,
|
||||
}),
|
||||
anchorMs: optionalNonNegativeIntegerSchema({
|
||||
description: "Start anchor ms (kind=every)",
|
||||
maximum: MAX_DATE_TIMESTAMP_MS,
|
||||
}),
|
||||
expr: Type.Optional(
|
||||
Type.String({
|
||||
@@ -116,7 +121,10 @@ function createCronScheduleSchema(): TSchema {
|
||||
'IANA timezone for wall-clock fields; missing=Gateway host local timezone. Example "Asia/Shanghai".',
|
||||
}),
|
||||
),
|
||||
staggerMs: optionalNonNegativeIntegerSchema({ description: "Jitter ms (kind=cron)" }),
|
||||
staggerMs: optionalNonNegativeIntegerSchema({
|
||||
description: "Jitter ms (kind=cron)",
|
||||
maximum: MAX_DATE_TIMESTAMP_MS,
|
||||
}),
|
||||
command: Type.Optional(
|
||||
Type.Array(Type.String({ minLength: 1 }), {
|
||||
minItems: 1,
|
||||
@@ -198,6 +206,19 @@ function cronDeliverySchema(params: { nullableClears: boolean }) {
|
||||
},
|
||||
{ additionalProperties: true },
|
||||
);
|
||||
const completionDestinationObject = Type.Object(
|
||||
{
|
||||
mode: Type.Literal("webhook"),
|
||||
to: Type.String({
|
||||
minLength: 1,
|
||||
description: "Completion webhook target; only valid with delivery.mode=announce",
|
||||
}),
|
||||
},
|
||||
{
|
||||
additionalProperties: true,
|
||||
description: "Additional completion webhook; requires delivery.mode=announce",
|
||||
},
|
||||
);
|
||||
|
||||
return Type.Optional(
|
||||
Type.Object(
|
||||
@@ -224,6 +245,14 @@ function cronDeliverySchema(params: { nullableClears: boolean }) {
|
||||
}),
|
||||
)
|
||||
: Type.Optional(failureDestinationObject),
|
||||
completionDestination: params.nullableClears
|
||||
? Type.Optional(
|
||||
Type.Union([completionDestinationObject, Type.Null()], {
|
||||
description:
|
||||
"Completion webhook destination; requires delivery.mode=announce; null clears.",
|
||||
}),
|
||||
)
|
||||
: Type.Optional(completionDestinationObject),
|
||||
},
|
||||
{ additionalProperties: true },
|
||||
),
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
findLlamacppGbnfSchemaViolations,
|
||||
normalizeToolParameterSchema,
|
||||
} from "@openclaw/ai/internal/openai";
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
// Cron tool schema tests cover the provider-facing parameter shape and runtime
|
||||
// validation compatibility for cron jobs.
|
||||
import { Value } from "typebox/value";
|
||||
@@ -205,16 +206,27 @@ describe("createCronToolSchema", () => {
|
||||
|
||||
it("advertises numeric cron params with runtime bounds", () => {
|
||||
for (const path of ["job.schedule.everyMs", "patch.schedule.everyMs"]) {
|
||||
expect(propertyAt(schemaRecord, path)).toMatchObject({ type: "integer", minimum: 1 });
|
||||
expect(propertyAt(schemaRecord, path)).toMatchObject({
|
||||
type: "integer",
|
||||
minimum: 1,
|
||||
maximum: MAX_DATE_TIMESTAMP_MS,
|
||||
});
|
||||
}
|
||||
for (const path of [
|
||||
"job.schedule.anchorMs",
|
||||
"job.schedule.staggerMs",
|
||||
"patch.schedule.anchorMs",
|
||||
"patch.schedule.staggerMs",
|
||||
"job.failureAlert.cooldownMs",
|
||||
"patch.failureAlert.cooldownMs",
|
||||
]) {
|
||||
for (const path of ["job.schedule.anchorMs", "patch.schedule.anchorMs"]) {
|
||||
expect(propertyAt(schemaRecord, path)).toMatchObject({
|
||||
type: "integer",
|
||||
minimum: 0,
|
||||
maximum: MAX_DATE_TIMESTAMP_MS,
|
||||
});
|
||||
}
|
||||
for (const path of ["job.schedule.staggerMs", "patch.schedule.staggerMs"]) {
|
||||
expect(propertyAt(schemaRecord, path)).toMatchObject({
|
||||
type: "integer",
|
||||
minimum: 0,
|
||||
maximum: MAX_DATE_TIMESTAMP_MS,
|
||||
});
|
||||
}
|
||||
for (const path of ["job.failureAlert.cooldownMs", "patch.failureAlert.cooldownMs"]) {
|
||||
expect(propertyAt(schemaRecord, path)).toMatchObject({ type: "integer", minimum: 0 });
|
||||
}
|
||||
for (const path of ["job.failureAlert.after", "patch.failureAlert.after"]) {
|
||||
@@ -247,18 +259,95 @@ describe("createCronToolSchema", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("job.delivery exposes mode, channel, to, threadId, bestEffort, accountId, failureDestination", () => {
|
||||
it("job.delivery exposes all supported delivery destinations", () => {
|
||||
expect(keysAt(schemaRecord, "job.delivery")).toEqual(
|
||||
[
|
||||
"accountId",
|
||||
"bestEffort",
|
||||
"channel",
|
||||
"completionDestination",
|
||||
"failureDestination",
|
||||
"mode",
|
||||
"threadId",
|
||||
"to",
|
||||
].toSorted(),
|
||||
);
|
||||
const jobCompletion = propertyAt(schemaRecord, "job.delivery.completionDestination");
|
||||
expect(keysAt(schemaRecord, "job.delivery.completionDestination")).toEqual(["mode", "to"]);
|
||||
expect(jobCompletion?.required).toEqual(["mode", "to"]);
|
||||
expect(propertyAt(schemaRecord, "job.delivery.completionDestination.to")).toMatchObject({
|
||||
type: "string",
|
||||
minLength: 1,
|
||||
});
|
||||
const patchCompletion = propertyAt(schemaRecord, "patch.delivery.completionDestination");
|
||||
const patchCompletionObject = (
|
||||
patchCompletion?.anyOf as Array<Record<string, unknown>> | undefined
|
||||
)?.find((entry) => entry.type === "object");
|
||||
expect(
|
||||
Object.keys(
|
||||
(patchCompletionObject?.properties as Record<string, unknown> | undefined) ?? {},
|
||||
).toSorted(),
|
||||
).toEqual(["mode", "to"]);
|
||||
expect(
|
||||
Value.Check(schema, {
|
||||
action: "add",
|
||||
job: {
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
payload: { kind: "agentTurn", message: "run" },
|
||||
delivery: {
|
||||
mode: "announce",
|
||||
completionDestination: { mode: "webhook", to: "https://example.invalid/done" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
Value.Check(schema, {
|
||||
action: "update",
|
||||
id: "job-1",
|
||||
patch: { delivery: { completionDestination: null } },
|
||||
}),
|
||||
).toBe(true);
|
||||
for (const completionDestination of [
|
||||
null,
|
||||
{},
|
||||
{ mode: "webhook" },
|
||||
{ to: "https://example.invalid/done" },
|
||||
{ mode: "announce", to: "https://example.invalid/done" },
|
||||
{ mode: "webhook", to: "" },
|
||||
"https://example.invalid/done",
|
||||
]) {
|
||||
expect(
|
||||
Value.Check(schema, {
|
||||
action: "add",
|
||||
job: {
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
payload: { kind: "agentTurn", message: "run" },
|
||||
delivery: { mode: "announce", completionDestination },
|
||||
},
|
||||
}),
|
||||
).toBe(false);
|
||||
}
|
||||
for (const providerSchema of [
|
||||
providerSchemaRecord,
|
||||
jjccGeminiSchemaRecord,
|
||||
llamacppSchemaRecord,
|
||||
]) {
|
||||
expect(propertyAt(providerSchema, "job.delivery.completionDestination")).toMatchObject({
|
||||
type: "object",
|
||||
required: ["mode", "to"],
|
||||
});
|
||||
}
|
||||
for (const providerSchema of [providerSchemaRecord, jjccGeminiSchemaRecord]) {
|
||||
expect(propertyAt(providerSchema, "patch.delivery.completionDestination")).toMatchObject({
|
||||
type: "object",
|
||||
required: ["mode", "to"],
|
||||
description: expect.stringContaining("null clears"),
|
||||
});
|
||||
}
|
||||
expect(
|
||||
propertyAt(llamacppSchemaRecord, "patch.delivery.completionDestination")?.anyOf,
|
||||
).toContainEqual({ type: "null" });
|
||||
});
|
||||
|
||||
it("job.payload exposes conversational and script payload fields", () => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Cron tool tests cover schedule guidance, scoped job operations, delivery
|
||||
// context inheritance, session routing, and agent id ownership.
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { callGatewayMock, extractDeliveryInfoMock } = vi.hoisted(() => ({
|
||||
@@ -1103,6 +1104,22 @@ describe("cron tool", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("canonicalizes the inclusive Date maximum without losing service compatibility", async () => {
|
||||
const tool = createTestCronTool();
|
||||
await tool.execute("call-date-max", {
|
||||
action: "add",
|
||||
job: {
|
||||
name: "far-future",
|
||||
schedule: { atMs: MAX_DATE_TIMESTAMP_MS },
|
||||
payload: { kind: "systemEvent", text: "hello" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(expectSingleGatewayCallMethod("cron.add")).toMatchObject({
|
||||
schedule: { kind: "at", at: new Date(MAX_DATE_TIMESTAMP_MS).toISOString() },
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves omitted declaration enablement and forwards explicit enablement", async () => {
|
||||
const tool = createTestCronTool();
|
||||
const baseJob = {
|
||||
|
||||
@@ -196,7 +196,7 @@ PACED LOOP: recurring job + pacing{min?,max?} durations ("15m","4h"; at least on
|
||||
|
||||
TRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call("exec",{command:"..."}).
|
||||
|
||||
DELIVERY {mode:"none"|"announce"|"webhook",channel?,to?,threadId?,bestEffort?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:"none". webhook posts finished-run event to URL in \`to\`.
|
||||
DELIVERY {mode:"none"|"announce"|"webhook",channel?,to?,threadId?,bestEffort?,completionDestination?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:"none". webhook posts finished-run event to URL in \`to\`. To keep announce delivery and also POST completion, use mode:"announce" with completionDestination:{mode:"webhook",to:"https://..."}.
|
||||
|
||||
Job wakeMode (main jobs): "now"(default)|"next-heartbeat". Restricted automation-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.`,
|
||||
parameters: createCronToolSchema(),
|
||||
|
||||
@@ -12,7 +12,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { heartbeatTaskDeclarationKey, isHeartbeatTaskCronJob } from "../cron/heartbeat-task.js";
|
||||
import { cronSchedulingInputsEqual } from "../cron/schedule-identity.js";
|
||||
import { readHeartbeatMonitorScratch } from "../cron/scratch-store.js";
|
||||
import { computeJobNextRunAtMs, hasScheduledNextRunAtMs } from "../cron/service/jobs.js";
|
||||
import { computeJobNextRunAtMs, hasScheduledNextRunAtMs } from "../cron/service/jobs-scheduling.js";
|
||||
import { resolveCronJobsStorePathFromConfig } from "../cron/store.js";
|
||||
import { cronStoreKey } from "../cron/store/key.js";
|
||||
import {
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
} from "../../../../packages/normalization-core/src/string-coerce.js";
|
||||
import { parseAbsoluteTimeMs } from "../../../cron/parse.js";
|
||||
import { getInvalidPersistedCronJobReason } from "../../../cron/persisted-shape.js";
|
||||
import { coerceFiniteScheduleNumber } from "../../../cron/schedule.js";
|
||||
import { coerceFiniteScheduleNumber } from "../../../cron/schedule-number.js";
|
||||
import { inferCronJobName } from "../../../cron/service/normalize.js";
|
||||
import { normalizeCronStaggerMs, resolveDefaultCronStaggerMs } from "../../../cron/stagger.js";
|
||||
import {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveCronJobConfigRevision } from "./config-revision.js";
|
||||
import { setupCronServiceSuite } from "./service.test-harness.js";
|
||||
@@ -152,7 +153,7 @@ describe("resolveCronJobConfigRevision", () => {
|
||||
{
|
||||
...makeJob(),
|
||||
id: "command-empty-env",
|
||||
schedule: { kind: "every", everyMs: Number.MAX_SAFE_INTEGER, anchorMs: 0 },
|
||||
schedule: { kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS, anchorMs: 0 },
|
||||
payload: { kind: "command", argv: ["true"], env: {}, input: "" },
|
||||
failureAlert: false,
|
||||
},
|
||||
|
||||
@@ -8,10 +8,6 @@ const channelPluginRuntimeLoader = createLazyImportLoader<ChannelPluginRuntime>(
|
||||
() => import("../../channels/plugins/index.js"),
|
||||
);
|
||||
|
||||
async function loadChannelPluginRuntime() {
|
||||
return await channelPluginRuntimeLoader.load();
|
||||
}
|
||||
|
||||
/** Resolves channel-specific cron output preferences from loaded channel plugins. */
|
||||
export async function resolveCronChannelOutputPolicy(
|
||||
channel: string | undefined,
|
||||
@@ -23,7 +19,7 @@ export async function resolveCronChannelOutputPolicy(
|
||||
if (!channelId) {
|
||||
return { preferFinalAssistantVisibleText: opts?.deliveryRequested !== true };
|
||||
}
|
||||
const { getChannelPlugin } = await loadChannelPluginRuntime();
|
||||
const { getChannelPlugin } = await channelPluginRuntimeLoader.load();
|
||||
return {
|
||||
preferFinalAssistantVisibleText:
|
||||
getChannelPlugin(channelId)?.outbound?.preferFinalAssistantVisibleText === true,
|
||||
@@ -43,7 +39,7 @@ export async function resolveCurrentChannelTarget(params: {
|
||||
if (!channelId) {
|
||||
return params.to;
|
||||
}
|
||||
const { getChannelPlugin } = await loadChannelPluginRuntime();
|
||||
const { getChannelPlugin } = await channelPluginRuntimeLoader.load();
|
||||
return (
|
||||
getChannelPlugin(channelId)?.threading?.resolveCurrentChannelId?.({
|
||||
to: params.to,
|
||||
|
||||
@@ -55,24 +55,6 @@ const outboundSessionRuntimeLoader = createLazyImportLoader(
|
||||
const transcriptRuntimeLoader = createLazyImportLoader(
|
||||
() => import("../../config/sessions/transcript.runtime.js"),
|
||||
);
|
||||
async function loadDeliveryOutboundRuntime(): Promise<
|
||||
typeof import("./delivery-outbound.runtime.js")
|
||||
> {
|
||||
return await deliveryOutboundRuntimeLoader.load();
|
||||
}
|
||||
|
||||
async function loadOutboundSessionRuntime(): Promise<
|
||||
typeof import("../../infra/outbound/outbound-session.js")
|
||||
> {
|
||||
return await outboundSessionRuntimeLoader.load();
|
||||
}
|
||||
|
||||
async function loadTranscriptRuntime(): Promise<
|
||||
typeof import("../../config/sessions/transcript.runtime.js")
|
||||
> {
|
||||
return await transcriptRuntimeLoader.load();
|
||||
}
|
||||
|
||||
export function shouldQueueCronAwareness(params: {
|
||||
job: CronJob;
|
||||
delivery: SuccessfulDeliveryTarget;
|
||||
@@ -191,7 +173,7 @@ export async function queueCronAwarenessSystemEvent(params: {
|
||||
targetText?: string;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const { enqueueSystemEvent } = await loadDeliveryOutboundRuntime();
|
||||
const { enqueueSystemEvent } = await deliveryOutboundRuntimeLoader.load();
|
||||
const mainSessionKey = resolveCronAwarenessMainSessionKey({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
@@ -344,7 +326,7 @@ async function resolveCronDeliveryRouteSessionKey(params: {
|
||||
}): Promise<string> {
|
||||
try {
|
||||
const { resolveOutboundSessionRoute, ensureOutboundSessionEntry } =
|
||||
await loadOutboundSessionRuntime();
|
||||
await outboundSessionRuntimeLoader.load();
|
||||
const route = await resolveOutboundSessionRoute({
|
||||
cfg: params.cfg,
|
||||
channel: params.delivery.channel,
|
||||
@@ -534,7 +516,7 @@ async function appendDirectCronDeliveryTranscriptMirror(params: {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const { appendAssistantMessageToSessionTranscript } = await loadTranscriptRuntime();
|
||||
const { appendAssistantMessageToSessionTranscript } = await transcriptRuntimeLoader.load();
|
||||
const result = await appendAssistantMessageToSessionTranscript(params.mirror);
|
||||
if (!result.ok) {
|
||||
await logCronDeliveryWarn(
|
||||
|
||||
@@ -24,10 +24,9 @@ import { stringifyRouteThreadId } from "../../plugin-sdk/channel-route.js";
|
||||
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
|
||||
import { shouldAttemptTtsPayload } from "../../tts/tts-config.js";
|
||||
import { createCronExecutionId } from "../run-id.js";
|
||||
import { hasScheduledNextRunAtMs } from "../service/jobs.js";
|
||||
import { hasScheduledNextRunAtMs } from "../service/jobs-scheduling.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import type { DeliveryTargetResolution } from "./delivery-target.js";
|
||||
import { cleanupCronRunSessionAfterRun } from "./session-cleanup.js";
|
||||
|
||||
type SuccessfulDeliveryTarget = Extract<DeliveryTargetResolution, { ok: true }>;
|
||||
|
||||
@@ -37,27 +36,6 @@ export const DIRECT_CRON_DELIVERY_COMPLETION_RETENTION = {
|
||||
maxEntries: 2_000,
|
||||
} as const satisfies DeliveryQueueCompletionRetention;
|
||||
|
||||
/** Deletes or retires ephemeral direct-delivery cron sessions for delete-after-run jobs. */
|
||||
export async function cleanupDirectCronSession(params: {
|
||||
job: CronJob;
|
||||
agentSessionKey: string;
|
||||
sessionId: string;
|
||||
lifecycleRevision: string;
|
||||
sessionUpdatedAt: number;
|
||||
beforeSessionDelete?: () => void;
|
||||
retireReason: string;
|
||||
}): Promise<void> {
|
||||
await cleanupCronRunSessionAfterRun({
|
||||
job: params.job,
|
||||
agentSessionKey: params.agentSessionKey,
|
||||
sessionId: params.sessionId,
|
||||
lifecycleRevision: params.lifecycleRevision,
|
||||
sessionUpdatedAt: params.sessionUpdatedAt,
|
||||
beforeDelete: params.beforeSessionDelete,
|
||||
reason: params.retireReason,
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeDeliveryTarget(channel: string, to: string): string {
|
||||
const toTrimmed = to.trim();
|
||||
return normalizeTargetForProvider(channel, toTrimmed) ?? toTrimmed;
|
||||
@@ -124,14 +102,6 @@ const deliverySubagentRegistryRuntimeLoader = createLazyImportLoader(
|
||||
() => import("./delivery-subagent-registry.runtime.js"),
|
||||
);
|
||||
|
||||
async function loadDeliveryLoggerRuntime(): Promise<typeof import("./delivery-logger.runtime.js")> {
|
||||
return await deliveryLoggerRuntimeLoader.load();
|
||||
}
|
||||
|
||||
async function loadTtsRuntime(): Promise<typeof import("../../tts/tts.runtime.js")> {
|
||||
return await ttsRuntimeLoader.load();
|
||||
}
|
||||
|
||||
export async function loadDeliverySubagentRegistryRuntime(): Promise<
|
||||
typeof import("./delivery-subagent-registry.runtime.js")
|
||||
> {
|
||||
@@ -139,17 +109,17 @@ export async function loadDeliverySubagentRegistryRuntime(): Promise<
|
||||
}
|
||||
|
||||
export async function logCronDeliveryWarn(message: string): Promise<void> {
|
||||
const { logWarn } = await loadDeliveryLoggerRuntime();
|
||||
const { logWarn } = await deliveryLoggerRuntimeLoader.load();
|
||||
logWarn(message);
|
||||
}
|
||||
|
||||
export async function logCronDeliveryError(message: string): Promise<void> {
|
||||
const { logError } = await loadDeliveryLoggerRuntime();
|
||||
const { logError } = await deliveryLoggerRuntimeLoader.load();
|
||||
logError(message);
|
||||
}
|
||||
|
||||
export function logCronDeliveryErrorDeferred(message: string): void {
|
||||
void loadDeliveryLoggerRuntime().then(({ logError }) => {
|
||||
void deliveryLoggerRuntimeLoader.load().then(({ logError }) => {
|
||||
logError(message);
|
||||
});
|
||||
}
|
||||
@@ -191,7 +161,7 @@ export async function maybeApplyTtsToCronPayloads(params: {
|
||||
) {
|
||||
return params.payloads;
|
||||
}
|
||||
const { maybeApplyTtsToPayload } = await loadTtsRuntime();
|
||||
const { maybeApplyTtsToPayload } = await ttsRuntimeLoader.load();
|
||||
return await Promise.all(
|
||||
params.payloads.map((payload) =>
|
||||
maybeApplyTtsToPayload({
|
||||
|
||||
@@ -35,7 +35,6 @@ import {
|
||||
} from "./delivery-dispatch-awareness.js";
|
||||
import {
|
||||
buildDirectCronDeliveryIdempotencyKey,
|
||||
cleanupDirectCronSession,
|
||||
DIRECT_CRON_DELIVERY_COMPLETION_RETENTION,
|
||||
isCompletedDirectCronDelivery,
|
||||
isStaleCronDelivery,
|
||||
@@ -70,23 +69,7 @@ const deliveryOutboundRuntimeLoader = createLazyImportLoader(
|
||||
const subagentFollowupRuntimeLoader = createLazyImportLoader(
|
||||
() => import("./subagent-followup.runtime.js"),
|
||||
);
|
||||
async function loadDeliveryOutboundRuntime(): Promise<
|
||||
typeof import("./delivery-outbound.runtime.js")
|
||||
> {
|
||||
return await deliveryOutboundRuntimeLoader.load();
|
||||
}
|
||||
|
||||
async function loadSubagentFollowupRuntime(): Promise<
|
||||
typeof import("./subagent-followup.runtime.js")
|
||||
> {
|
||||
return await subagentFollowupRuntimeLoader.load();
|
||||
}
|
||||
|
||||
export {
|
||||
cleanupDirectCronSession,
|
||||
queueCronMessageToolDeliveryAwareness,
|
||||
resolveCronDeliveryBestEffort,
|
||||
};
|
||||
export { queueCronMessageToolDeliveryAwareness, resolveCronDeliveryBestEffort };
|
||||
/** Dispatches cron run output through verified message-tool or direct delivery paths. */
|
||||
export async function dispatchCronDelivery(
|
||||
params: DispatchCronDeliveryParams,
|
||||
@@ -201,7 +184,7 @@ export async function dispatchCronDelivery(
|
||||
createOutboundSendDeps,
|
||||
resolveAgentOutboundIdentity,
|
||||
sendDurableMessageBatch,
|
||||
} = await loadDeliveryOutboundRuntime();
|
||||
} = await deliveryOutboundRuntimeLoader.load();
|
||||
const identity = resolveAgentOutboundIdentity(params.cfgWithAgentDefaults, params.agentId);
|
||||
try {
|
||||
const summaryFallbackText = resolveDirectCronSummaryFallbackText({
|
||||
@@ -590,7 +573,7 @@ export async function dispatchCronDelivery(
|
||||
const needsSubagentFollowupRuntime =
|
||||
shouldCheckCompletedDescendants || activeSubagentRuns > 0 || expectedSubagentFollowup;
|
||||
const subagentFollowupRuntime = needsSubagentFollowupRuntime
|
||||
? await loadSubagentFollowupRuntime()
|
||||
? await subagentFollowupRuntimeLoader.load()
|
||||
: undefined;
|
||||
// Also check for already-completed descendants. If the subagent finished
|
||||
// before delivery-dispatch runs, activeSubagentRuns is 0 and
|
||||
@@ -712,7 +695,7 @@ export async function dispatchCronDelivery(
|
||||
// inherited shared-bucket target was refused). We never send here, so a
|
||||
// deleteAfterRun cron must still retire its session/transcript before
|
||||
// returning — otherwise the one-shot session leaks. Safe no-op for
|
||||
// non-deleteAfterRun / non-cron sessions (see cleanupDirectCronSession).
|
||||
// Cleanup is a no-op for non-deleteAfterRun or non-cron sessions.
|
||||
await cleanupDirectCronSessionIfNeeded();
|
||||
if (!params.deliveryBestEffort) {
|
||||
return buildDeliveryState(failDeliveryTarget(params.resolvedDelivery.error.message));
|
||||
|
||||
@@ -45,10 +45,6 @@ const targetsRuntimeLoader = createLazyImportLoader(
|
||||
() => import("../../infra/outbound/targets.runtime.js"),
|
||||
);
|
||||
|
||||
async function loadTargetsRuntime() {
|
||||
return await targetsRuntimeLoader.load();
|
||||
}
|
||||
|
||||
async function resolveOutboundTargetWithRuntime(
|
||||
params: Parameters<typeof tryResolveLoadedOutboundTarget>[0],
|
||||
) {
|
||||
@@ -57,7 +53,7 @@ async function resolveOutboundTargetWithRuntime(
|
||||
if (loaded) {
|
||||
return loaded;
|
||||
}
|
||||
const { resolveOutboundTarget } = await loadTargetsRuntime();
|
||||
const { resolveOutboundTarget } = await targetsRuntimeLoader.load();
|
||||
return resolveOutboundTarget({ ...params, allowBootstrap: true });
|
||||
} catch (err) {
|
||||
return {
|
||||
@@ -74,14 +70,6 @@ const deliveryTargetRuntimeLoader = createLazyImportLoader(
|
||||
() => import("./delivery-target.runtime.js"),
|
||||
);
|
||||
|
||||
async function loadChannelSelectionRuntime() {
|
||||
return await channelSelectionRuntimeLoader.load();
|
||||
}
|
||||
|
||||
async function loadDeliveryTargetRuntime() {
|
||||
return await deliveryTargetRuntimeLoader.load();
|
||||
}
|
||||
|
||||
function isNonEmptyThreadId(value: string | number | undefined | null): value is string | number {
|
||||
return value != null && value !== "";
|
||||
}
|
||||
@@ -146,7 +134,7 @@ export async function resolveDeliveryTarget(
|
||||
const requestedChannel = typeof jobPayload.channel === "string" ? jobPayload.channel : "last";
|
||||
const explicitTo = typeof jobPayload.to === "string" ? jobPayload.to : undefined;
|
||||
const allowMismatchedLastTo = requestedChannel === "last";
|
||||
const deliveryTargetRuntime = await loadDeliveryTargetRuntime();
|
||||
const deliveryTargetRuntime = await deliveryTargetRuntimeLoader.load();
|
||||
|
||||
const sessionCfg = cfg.session;
|
||||
const mainSessionKey = resolveAgentMainSessionKey({ cfg, agentId });
|
||||
@@ -199,7 +187,7 @@ export async function resolveDeliveryTarget(
|
||||
fallbackChannel = preliminary.lastChannel;
|
||||
} else {
|
||||
try {
|
||||
const { resolveMessageChannelSelection } = await loadChannelSelectionRuntime();
|
||||
const { resolveMessageChannelSelection } = await channelSelectionRuntimeLoader.load();
|
||||
const selection = await resolveMessageChannelSelection({ cfg });
|
||||
fallbackChannel = selection.channel;
|
||||
} catch (err) {
|
||||
|
||||
@@ -46,14 +46,6 @@ async function loadCodexNativeWebSearch() {
|
||||
return await codexNativeWebSearchLoader.load();
|
||||
}
|
||||
|
||||
async function loadWebToolRuntimeContext() {
|
||||
return await webToolRuntimeContextLoader.load();
|
||||
}
|
||||
|
||||
async function loadWebSearchRuntime() {
|
||||
return await webSearchRuntimeLoader.load();
|
||||
}
|
||||
|
||||
type CronDeliveryRuntime = typeof import("./run-delivery.runtime.js");
|
||||
export type ResolvedCronDeliveryTarget = Awaited<
|
||||
ReturnType<CronDeliveryRuntime["resolveDeliveryTarget"]>
|
||||
@@ -213,14 +205,14 @@ export async function createCronToolsAllowPreflightDiagnostics(params: {
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const { resolveWebSearchToolRuntimeContext } = await loadWebToolRuntimeContext();
|
||||
const { resolveWebSearchToolRuntimeContext } = await webToolRuntimeContextLoader.load();
|
||||
const { config, preferRuntimeProviders, runtimeWebSearch } = resolveWebSearchToolRuntimeContext(
|
||||
{
|
||||
config: params.cfg,
|
||||
lateBindRuntimeConfig: true,
|
||||
},
|
||||
);
|
||||
const { hasUsableWebSearchProvider } = await loadWebSearchRuntime();
|
||||
const { hasUsableWebSearchProvider } = await webSearchRuntimeLoader.load();
|
||||
const hasWebSearchProvider = hasUsableWebSearchProvider({
|
||||
config,
|
||||
agentDir: params.agentDir,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// Runtime delivery seam for isolated cron agent run orchestration.
|
||||
export { resolveDeliveryTarget } from "./delivery-target.js";
|
||||
export {
|
||||
cleanupDirectCronSession,
|
||||
dispatchCronDelivery,
|
||||
queueCronMessageToolDeliveryAwareness,
|
||||
resolveCronDeliveryBestEffort,
|
||||
|
||||
@@ -23,15 +23,11 @@ const cronExecutionCliRuntimeLoader = createLazyImportLoader(
|
||||
() => import("./run-execution-cli.runtime.js"),
|
||||
);
|
||||
|
||||
async function loadCronExecutionCliRuntime() {
|
||||
return await cronExecutionCliRuntimeLoader.load();
|
||||
}
|
||||
|
||||
/** Lazily resolves complete CLI bindings so cron continuations preserve reuse metadata. */
|
||||
export async function getCliSessionBinding(
|
||||
...args: Parameters<typeof import("../../agents/cli-session.js").getCliSessionBinding>
|
||||
): Promise<ReturnType<typeof import("../../agents/cli-session.js").getCliSessionBinding>> {
|
||||
const runtime = await loadCronExecutionCliRuntime();
|
||||
const runtime = await cronExecutionCliRuntimeLoader.load();
|
||||
return runtime.getCliSessionBinding(...args);
|
||||
}
|
||||
|
||||
@@ -39,6 +35,6 @@ export async function getCliSessionBinding(
|
||||
export async function runCliAgent(
|
||||
...args: Parameters<typeof import("../../agents/cli-runner.js").runCliAgent>
|
||||
): ReturnType<typeof import("../../agents/cli-runner.js").runCliAgent> {
|
||||
const runtime = await loadCronExecutionCliRuntime();
|
||||
const runtime = await cronExecutionCliRuntimeLoader.load();
|
||||
return runtime.runCliAgent(...args);
|
||||
}
|
||||
|
||||
@@ -79,14 +79,6 @@ const cronSubagentRegistryRuntimeLoader = createLazyImportLoader<CronSubagentReg
|
||||
() => import("./run-subagent-registry.runtime.js"),
|
||||
);
|
||||
|
||||
async function loadCronEmbeddedRuntime() {
|
||||
return await cronEmbeddedRuntimeLoader.load();
|
||||
}
|
||||
|
||||
async function loadCronSubagentRegistryRuntime() {
|
||||
return await cronSubagentRegistryRuntimeLoader.load();
|
||||
}
|
||||
|
||||
function hasCliSessionReuseMetadata(binding: CliSessionBinding): boolean {
|
||||
return Object.entries(binding).some(([key, value]) => key !== "sessionId" && value !== undefined);
|
||||
}
|
||||
@@ -582,7 +574,7 @@ function createCronPromptExecutor(params: {
|
||||
acceptedContextEngineTurnCandidate = contextEngineTurnCandidate;
|
||||
return result;
|
||||
}
|
||||
const { resolveFastModeState, runEmbeddedAgent } = await loadCronEmbeddedRuntime();
|
||||
const { resolveFastModeState, runEmbeddedAgent } = await cronEmbeddedRuntimeLoader.load();
|
||||
const promptCacheKey = resolveIsolatedCronPromptCacheKey({
|
||||
job: params.job,
|
||||
agentId: params.agentId,
|
||||
@@ -929,7 +921,7 @@ export async function executeCronRun(params: {
|
||||
let hasActiveDescendants = false;
|
||||
if (shouldRetryInterimAck) {
|
||||
const { countActiveDescendantRuns, listDescendantRunsForRequester } =
|
||||
await loadCronSubagentRegistryRuntime();
|
||||
await cronSubagentRegistryRuntimeLoader.load();
|
||||
hasFreshDescendants = listDescendantRunsForRequester(params.runSessionKey).some((entry) => {
|
||||
const descendantStartedAt =
|
||||
typeof entry.execution.startedAt === "number"
|
||||
|
||||
@@ -32,28 +32,17 @@ import {
|
||||
setSessionRuntimeModel,
|
||||
} from "./run.runtime.js";
|
||||
import type { RunCronAgentTurnResult } from "./run.types.js";
|
||||
import { cleanupCronRunSessionAfterRun } from "./session-cleanup.js";
|
||||
|
||||
type CronExecutionRuntime = typeof import("./run-executor.runtime.js");
|
||||
type CronExecutionResult = Awaited<ReturnType<CronExecutionRuntime["executeCronRun"]>>;
|
||||
|
||||
const cronContextRuntimeLoader = createLazyImportLoader(() => import("./run-context.runtime.js"));
|
||||
|
||||
async function loadCronContextRuntime() {
|
||||
return await cronContextRuntimeLoader.load();
|
||||
}
|
||||
|
||||
function resolvePositiveContextTokens(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
async function loadCliRunnerRuntime() {
|
||||
return await import("../../agents/cli-runner.runtime.js");
|
||||
}
|
||||
|
||||
async function loadUsageFormatRuntime() {
|
||||
return await import("../../utils/usage-format.js");
|
||||
}
|
||||
|
||||
export async function finalizeCronRun(params: {
|
||||
prepared: PreparedCronRunContext;
|
||||
execution: CronExecutionResult;
|
||||
@@ -66,6 +55,18 @@ export async function finalizeCronRun(params: {
|
||||
const finalRunResult = execution.runResult;
|
||||
const payloads = finalRunResult.payloads ?? [];
|
||||
let telemetry: CronRunTelemetry | undefined;
|
||||
const cleanupRunSession = async (reason: string) => {
|
||||
await cleanupCronRunSessionAfterRun({
|
||||
job: prepared.input.job,
|
||||
agentSessionKey: prepared.agentSessionKey,
|
||||
sessionId: prepared.currentRunSessionId(),
|
||||
lifecycleRevision: prepared.cronSession.lifecycleRevision,
|
||||
sessionUpdatedAt: prepared.cronSession.sessionEntry.updatedAt,
|
||||
beforeDelete: params.beforeSessionDelete,
|
||||
reason,
|
||||
});
|
||||
params.markCronRunSessionCleanupAttempted();
|
||||
};
|
||||
|
||||
// Late aborted results may still contain billable usage. Recheck before each
|
||||
// metadata mutation because lazy runtime loads below can yield to the timeout.
|
||||
@@ -93,7 +94,7 @@ export async function finalizeCronRun(params: {
|
||||
execution.liveSelection.provider;
|
||||
const contextTokens =
|
||||
resolvePositiveContextTokens(prepared.agentCfg?.contextTokens) ??
|
||||
(await loadCronContextRuntime()).lookupContextTokens(modelUsed, {
|
||||
(await cronContextRuntimeLoader.load()).lookupContextTokens(modelUsed, {
|
||||
allowAsyncLoad: false,
|
||||
}) ??
|
||||
resolvePositiveContextTokens(prepared.cronSession.sessionEntry.contextTokens) ??
|
||||
@@ -109,19 +110,20 @@ export async function finalizeCronRun(params: {
|
||||
const cliSessionBinding = finalRunResult.meta?.agentMeta?.cliSessionBinding;
|
||||
const cliSessionId = finalRunResult.meta?.agentMeta?.sessionId?.trim();
|
||||
if (finalRunResult.meta?.agentMeta?.clearCliSessionBinding === true) {
|
||||
const { clearCliSession } = await loadCliRunnerRuntime();
|
||||
const { clearCliSession } = await import("../../agents/cli-runner.runtime.js");
|
||||
clearCliSession(prepared.cronSession.sessionEntry, providerUsed);
|
||||
} else if (cliSessionBinding?.sessionId?.trim()) {
|
||||
const { setCliSessionBinding } = await loadCliRunnerRuntime();
|
||||
const { setCliSessionBinding } = await import("../../agents/cli-runner.runtime.js");
|
||||
setCliSessionBinding(prepared.cronSession.sessionEntry, providerUsed, cliSessionBinding);
|
||||
} else if (cliSessionId) {
|
||||
const { setCliSessionId } = await loadCliRunnerRuntime();
|
||||
const { setCliSessionId } = await import("../../agents/cli-runner.runtime.js");
|
||||
setCliSessionId(prepared.cronSession.sessionEntry, providerUsed, cliSessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hasNonzeroUsage(usage)) {
|
||||
const { estimateUsageCost, resolveModelCostConfig } = await loadUsageFormatRuntime();
|
||||
const { estimateUsageCost, resolveModelCostConfig } =
|
||||
await import("../../utils/usage-format.js");
|
||||
const input = usage.input ?? 0;
|
||||
const output = usage.output ?? 0;
|
||||
const cacheRead = usage.cacheRead ?? 0;
|
||||
@@ -271,17 +273,7 @@ export async function finalizeCronRun(params: {
|
||||
if (finalRunResult.meta?.aborted === true && !cronPayloadOutcome.hasFatalErrorPayload) {
|
||||
const metaErrorMessage = normalizeOptionalString(finalRunResult.meta.error?.message);
|
||||
const error = metaErrorMessage ?? "cron isolated agent run aborted";
|
||||
const { cleanupDirectCronSession } = await loadCronDeliveryRuntime();
|
||||
await cleanupDirectCronSession({
|
||||
job: prepared.input.job,
|
||||
agentSessionKey: prepared.agentSessionKey,
|
||||
sessionId: prepared.currentRunSessionId(),
|
||||
lifecycleRevision: prepared.cronSession.lifecycleRevision,
|
||||
sessionUpdatedAt: prepared.cronSession.sessionEntry.updatedAt,
|
||||
beforeSessionDelete: params.beforeSessionDelete,
|
||||
retireReason: "cron-delete-after-run-aborted",
|
||||
});
|
||||
params.markCronRunSessionCleanupAttempted();
|
||||
await cleanupRunSession("cron-delete-after-run-aborted");
|
||||
return prepared.withRunSession({
|
||||
status: "error",
|
||||
error,
|
||||
@@ -421,17 +413,7 @@ export async function finalizeCronRun(params: {
|
||||
if (hasFatalStructuredErrorPayload && prepared.deliveryRequested) {
|
||||
// Structured run error payloads belong in cron state and failure alerts,
|
||||
// not the normal completion announce path where provider JSON can leak.
|
||||
const { cleanupDirectCronSession } = await loadCronDeliveryRuntime();
|
||||
await cleanupDirectCronSession({
|
||||
job: prepared.input.job,
|
||||
agentSessionKey: prepared.agentSessionKey,
|
||||
sessionId: prepared.currentRunSessionId(),
|
||||
lifecycleRevision: prepared.cronSession.lifecycleRevision,
|
||||
sessionUpdatedAt: prepared.cronSession.sessionEntry.updatedAt,
|
||||
beforeSessionDelete: params.beforeSessionDelete,
|
||||
retireReason: "cron-delete-after-run-fatal-error",
|
||||
});
|
||||
params.markCronRunSessionCleanupAttempted();
|
||||
await cleanupRunSession("cron-delete-after-run-fatal-error");
|
||||
const deliveryTrace = buildCronDeliveryTrace({
|
||||
deliveryPlan: prepared.deliveryPlan,
|
||||
resolvedDelivery: prepared.resolvedDelivery,
|
||||
|
||||
@@ -8,8 +8,8 @@ import type { CronDeliveryMode } from "../types.js";
|
||||
import type { MutableCronSession } from "./run-session-state.js";
|
||||
import {
|
||||
buildSafeExternalPromptMock,
|
||||
callGatewayMock,
|
||||
clearFastTestEnv,
|
||||
cleanupDirectCronSessionMock,
|
||||
dispatchCronDeliveryMock,
|
||||
getChannelPluginMock,
|
||||
isCliProviderMock,
|
||||
@@ -1224,15 +1224,7 @@ describe("runCronIsolatedAgentTurn message tool policy", () => {
|
||||
expect(result.delivered).toBe(false);
|
||||
expect(result.deliveryAttempted).toBe(false);
|
||||
expect(dispatchCronDeliveryMock).not.toHaveBeenCalled();
|
||||
expect(cleanupDirectCronSessionMock).toHaveBeenCalledWith({
|
||||
job: expect.objectContaining({ id: "fatal-error-payload" }),
|
||||
agentSessionKey: "agent:default:cron:message-tool-policy",
|
||||
sessionId: "test-session-id",
|
||||
lifecycleRevision: "test-lifecycle-revision",
|
||||
sessionUpdatedAt: expect.any(Number),
|
||||
beforeSessionDelete: expect.any(Function),
|
||||
retireReason: "cron-delete-after-run-fatal-error",
|
||||
});
|
||||
expect(callGatewayMock).not.toHaveBeenCalled();
|
||||
expectDeliveryFields(result.delivery, {
|
||||
intended: { channel: "messagechat", to: "123", source: "explicit" },
|
||||
resolved: { ok: true, channel: "messagechat", to: "123", source: "explicit" },
|
||||
@@ -1260,18 +1252,7 @@ describe("runCronIsolatedAgentTurn message tool policy", () => {
|
||||
});
|
||||
|
||||
expect(dispatchCronDeliveryMock).not.toHaveBeenCalled();
|
||||
expect(cleanupDirectCronSessionMock).toHaveBeenCalledWith({
|
||||
job: expect.objectContaining({
|
||||
id: "fatal-delete-after-run",
|
||||
deleteAfterRun: true,
|
||||
}),
|
||||
agentSessionKey: "agent:default:cron:message-tool-policy",
|
||||
sessionId: "test-session-id",
|
||||
lifecycleRevision: "test-lifecycle-revision",
|
||||
sessionUpdatedAt: expect.any(Number),
|
||||
beforeSessionDelete: expect.any(Function),
|
||||
retireReason: "cron-delete-after-run-fatal-error",
|
||||
});
|
||||
expect(callGatewayMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("skips cron fallback delivery when the message tool already sent to the same target", async () => {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";
|
||||
import { makeIsolatedAgentJobFixture, makeIsolatedAgentParamsFixture } from "./job-fixtures.js";
|
||||
import { setupRunCronIsolatedAgentTurnSuite } from "./run.suite-helpers.js";
|
||||
import {
|
||||
cleanupDirectCronSessionMock,
|
||||
callGatewayMock,
|
||||
dispatchCronDeliveryMock,
|
||||
loadRunCronIsolatedAgentTurn,
|
||||
resolveCronDeliveryPlanMock,
|
||||
@@ -129,15 +129,7 @@ describe("runCronIsolatedAgentTurn - meta.error status propagation", () => {
|
||||
|
||||
expect(result.status).toBe("error");
|
||||
expect(result.error).toBe("cron isolated agent run aborted");
|
||||
expect(cleanupDirectCronSessionMock).toHaveBeenCalledWith({
|
||||
job: expect.objectContaining({ deleteAfterRun: true }),
|
||||
agentSessionKey: "agent:default:cron:test",
|
||||
sessionId: "test-session-id",
|
||||
lifecycleRevision: "test-lifecycle-revision",
|
||||
sessionUpdatedAt: expect.any(Number),
|
||||
beforeSessionDelete: expect.any(Function),
|
||||
retireReason: "cron-delete-after-run-aborted",
|
||||
});
|
||||
expect(callGatewayMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("marks a completed embedded run with no final payload as a cron error", async () => {
|
||||
|
||||
@@ -90,7 +90,6 @@ export const resolveCronDeliveryPlanMock = createMock();
|
||||
export const resolveDeliveryTargetMock = createMock();
|
||||
export const dispatchCronDeliveryMock = createMock();
|
||||
export const queueCronMessageToolDeliveryAwarenessMock = createMock();
|
||||
export const cleanupDirectCronSessionMock = createMock();
|
||||
export const preflightCronModelProviderMock = createMock();
|
||||
export const resolveSessionAuthProfileOverrideMock = createMock();
|
||||
export const resolveFastModeStateMock = createMock();
|
||||
@@ -397,7 +396,6 @@ vi.mock("./run-delivery.runtime.js", async () => {
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
cleanupDirectCronSession: cleanupDirectCronSessionMock,
|
||||
resolveDeliveryTarget: resolveDeliveryTargetMock,
|
||||
dispatchCronDelivery: dispatchCronDeliveryMock,
|
||||
queueCronMessageToolDeliveryAwareness: queueCronMessageToolDeliveryAwarenessMock,
|
||||
@@ -760,8 +758,6 @@ function resetRunOutcomeMocks(): void {
|
||||
);
|
||||
queueCronMessageToolDeliveryAwarenessMock.mockReset();
|
||||
queueCronMessageToolDeliveryAwarenessMock.mockResolvedValue(undefined);
|
||||
cleanupDirectCronSessionMock.mockReset();
|
||||
cleanupDirectCronSessionMock.mockResolvedValue(undefined);
|
||||
preflightCronModelProviderMock.mockReset();
|
||||
preflightCronModelProviderMock.mockResolvedValue({ status: "available" });
|
||||
resolveSessionAuthProfileOverrideMock.mockReset();
|
||||
|
||||
@@ -40,10 +40,6 @@ import { cleanupCronRunSessionAfterRun } from "./session-cleanup.js";
|
||||
|
||||
const cronExecutorRuntimeLoader = createLazyImportLoader(() => import("./run-executor.runtime.js"));
|
||||
|
||||
async function loadCronExecutorRuntime() {
|
||||
return await cronExecutorRuntimeLoader.load();
|
||||
}
|
||||
|
||||
function isCronNestedLaneTaskTimeoutError(err: unknown): boolean {
|
||||
return isCommandLaneTaskTimeoutError(err, CommandLane.CronNested);
|
||||
}
|
||||
@@ -211,7 +207,7 @@ export async function runCronIsolatedAgentTurn(params: {
|
||||
bindAgentRunTaskRunId(initialSessionId, runContextOwnerToken, taskRunId);
|
||||
}
|
||||
}
|
||||
const { executeCronRun } = await loadCronExecutorRuntime();
|
||||
const { executeCronRun } = await cronExecutorRuntimeLoader.load();
|
||||
const executionParams: Parameters<typeof executeCronRun>[0] = {
|
||||
cfg: params.cfg,
|
||||
cfgWithAgentDefaults: prepared.context.cfgWithAgentDefaults,
|
||||
|
||||
@@ -8,10 +8,6 @@ const gatewayCallRuntimeLoader = createLazyImportLoader(
|
||||
() => import("../../gateway/call.runtime.js"),
|
||||
);
|
||||
|
||||
async function loadGatewayCallRuntime(): Promise<typeof import("../../gateway/call.runtime.js")> {
|
||||
return await gatewayCallRuntimeLoader.load();
|
||||
}
|
||||
|
||||
export type CronRunSessionCleanupOutcome =
|
||||
| "not-requested"
|
||||
| "deleted"
|
||||
@@ -33,7 +29,7 @@ export async function cleanupCronRunSessionAfterRun(params: {
|
||||
}
|
||||
params.beforeDelete?.();
|
||||
try {
|
||||
const { callGateway } = await loadGatewayCallRuntime();
|
||||
const { callGateway } = await gatewayCallRuntimeLoader.load();
|
||||
const result = await callGateway<{ deleted?: boolean }>({
|
||||
method: "sessions.delete",
|
||||
params: {
|
||||
|
||||
@@ -178,7 +178,6 @@ export function resolveCronSession(params: {
|
||||
let isNewSession: boolean;
|
||||
let systemSent: boolean;
|
||||
let resetBoundaryPending: { reason: "cron-stale"; sessionFile: string } | undefined;
|
||||
let staleBoundaryReset = false;
|
||||
|
||||
if (!params.forceNew && entry?.sessionId) {
|
||||
// Cron/webhook sessions follow the direct reset policy so scheduled turns
|
||||
@@ -211,7 +210,6 @@ export function resolveCronSession(params: {
|
||||
isNewSession = true;
|
||||
systemSent = false;
|
||||
if (!sourceSessionDiffers) {
|
||||
staleBoundaryReset = true;
|
||||
resetBoundaryPending = { reason: "cron-stale", sessionFile: params.sessionKey };
|
||||
}
|
||||
}
|
||||
@@ -222,7 +220,7 @@ export function resolveCronSession(params: {
|
||||
}
|
||||
|
||||
const previousSessionId =
|
||||
isNewSession && !sourceSessionDiffers && !staleBoundaryReset ? entry?.sessionId : undefined;
|
||||
isNewSession && !sourceSessionDiffers && !resetBoundaryPending ? entry?.sessionId : undefined;
|
||||
clearBootstrapSnapshotOnSessionRollover({
|
||||
sessionKey: params.sessionKey,
|
||||
previousSessionId,
|
||||
|
||||
+13
-11
@@ -1,3 +1,4 @@
|
||||
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
|
||||
import { parseDurationMs } from "../cli/parse-duration.js";
|
||||
import type { CronPacing } from "./types.js";
|
||||
|
||||
@@ -8,16 +9,15 @@ type CronPacingBounds = {
|
||||
};
|
||||
|
||||
function parsePositivePacingDuration(value: string, field: "min" | "max"): number {
|
||||
let durationMs: number;
|
||||
try {
|
||||
durationMs = parseDurationMs(value);
|
||||
const durationMs = parseDurationMs(value);
|
||||
if (durationMs > 0) {
|
||||
return durationMs;
|
||||
}
|
||||
} catch {
|
||||
throw new Error(`cron pacing ${field} must be a positive duration`);
|
||||
// Normalize parser details into the cron configuration contract below.
|
||||
}
|
||||
if (durationMs <= 0) {
|
||||
throw new Error(`cron pacing ${field} must be a positive duration`);
|
||||
}
|
||||
return durationMs;
|
||||
throw new Error(`cron pacing ${field} must be a positive duration`);
|
||||
}
|
||||
|
||||
/** Validates pacing strings and returns their millisecond bounds. */
|
||||
@@ -40,11 +40,13 @@ export function resolvePacedNextRunAtMs(params: {
|
||||
nowMs: number;
|
||||
delayMs: number;
|
||||
pacing: CronPacing;
|
||||
}): number {
|
||||
}): number | undefined {
|
||||
const { minMs, maxMs } = parseCronPacingBounds(params.pacing);
|
||||
const proposedAtMs = params.nowMs + params.delayMs;
|
||||
return Math.min(
|
||||
params.nowMs + (maxMs ?? Number.POSITIVE_INFINITY),
|
||||
Math.max(params.nowMs + (minMs ?? 0), proposedAtMs),
|
||||
return asDateTimestampMs(
|
||||
Math.min(
|
||||
params.nowMs + (maxMs ?? Number.POSITIVE_INFINITY),
|
||||
Math.max(params.nowMs + (minMs ?? 0), proposedAtMs),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -220,6 +220,8 @@ describe("parseAbsoluteTimeMs", () => {
|
||||
// JavaScript Date range ends at +100,000,000 days
|
||||
const maxValid = new Date(8640000000000000).getTime();
|
||||
expect(parseAbsoluteTimeMs(maxValid.toString())).toBe(maxValid);
|
||||
expect(parseAbsoluteTimeMs(new Date(maxValid).toISOString())).toBe(maxValid);
|
||||
expect(parseAbsoluteTimeMs("+275760-09-13T00:00:00.001Z")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+2
-2
@@ -3,8 +3,8 @@ import { parseStrictPositiveInteger } from "../infra/parse-finite-number.js";
|
||||
import { hasValidIsoCalendarComponents } from "../shared/iso-time.js";
|
||||
|
||||
const ISO_TZ_RE = /(Z|[+-]\d{2}:?\d{2})$/i;
|
||||
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
||||
const ISO_DATE_TIME_RE = /^\d{4}-\d{2}-\d{2}[Tt]/;
|
||||
const ISO_DATE_RE = /^(?:[+-]\d{6}|\d{4})-\d{2}-\d{2}$/;
|
||||
const ISO_DATE_TIME_RE = /^(?:[+-]\d{6}|\d{4})-\d{2}-\d{2}[Tt]/;
|
||||
|
||||
function normalizeUtcIso(raw: string) {
|
||||
if (ISO_TZ_RE.test(raw)) {
|
||||
|
||||
+81
-21
@@ -1,12 +1,62 @@
|
||||
import { compileSafeRegex } from "../security/safe-regex.js";
|
||||
/** Validates persisted cron job records before loading them from disk/state. */
|
||||
import {
|
||||
asSafeIntegerInRange,
|
||||
MAX_DATE_TIMESTAMP_MS,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
import { asRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { compileSafeRegex } from "../security/safe-regex.js";
|
||||
import { parseAbsoluteTimeMs } from "./parse.js";
|
||||
import type { CronJobState } from "./types.js";
|
||||
|
||||
const CRON_STATE_TIMESTAMP_FIELDS = [
|
||||
"nextRunAtMs",
|
||||
"scheduleActivatedAtMs",
|
||||
"startupCatchupAtMs",
|
||||
"pacedNextRunAtMs",
|
||||
"forcePreservedNextRunAtMs",
|
||||
"queuedAtMs",
|
||||
"runningAtMs",
|
||||
"lastRunAtMs",
|
||||
"lastFailureAlertAtMs",
|
||||
"lastTriggerEvalAtMs",
|
||||
"lastTriggerFireAtMs",
|
||||
"streamLastStartedAtMs",
|
||||
"streamLastExitAtMs",
|
||||
] as const satisfies readonly (keyof CronJobState)[];
|
||||
|
||||
function isValidStateTimestamp(value: unknown): boolean {
|
||||
return asSafeIntegerInRange(value, { min: 0, max: MAX_DATE_TIMESTAMP_MS }) !== undefined;
|
||||
}
|
||||
|
||||
function getInvalidCronJobStateTimestampField(state: unknown): string | undefined {
|
||||
const record = asRecord(state);
|
||||
const field = CRON_STATE_TIMESTAMP_FIELDS.find(
|
||||
(key) => record[key] !== undefined && !isValidStateTimestamp(record[key]),
|
||||
);
|
||||
if (field) {
|
||||
return field;
|
||||
}
|
||||
const atMs = asRecord(record.autoDisabled).atMs;
|
||||
return atMs !== undefined && !isValidStateTimestamp(atMs) ? "autoDisabled.atMs" : undefined;
|
||||
}
|
||||
|
||||
/** Rejects caller-authored state timestamps that cannot round-trip through Date and SQLite. */
|
||||
export function assertCronJobStateTimestamps(state: Partial<CronJobState>): void {
|
||||
const invalidField = getInvalidCronJobStateTimestampField(state);
|
||||
if (invalidField) {
|
||||
throw new Error(
|
||||
`cron state.${invalidField} must be a non-negative Date-valid integer timestamp`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Structural rejection code for persisted cron jobs that cannot be loaded safely. */
|
||||
type InvalidPersistedCronJobReason =
|
||||
| "missing-id"
|
||||
| "missing-schedule"
|
||||
| "invalid-schedule"
|
||||
| "invalid-state"
|
||||
| "unsatisfiable-schedule"
|
||||
| "invalid-trigger"
|
||||
| "missing-payload"
|
||||
| "invalid-payload";
|
||||
@@ -19,6 +69,9 @@ export function getInvalidPersistedCronJobReason(
|
||||
if (typeof id !== "string" || !id.trim()) {
|
||||
return "missing-id";
|
||||
}
|
||||
if (getInvalidCronJobStateTimestampField(candidate.state)) {
|
||||
return "invalid-state";
|
||||
}
|
||||
const schedule = candidate.schedule;
|
||||
if (!schedule || Array.isArray(schedule)) {
|
||||
return "missing-schedule";
|
||||
@@ -50,13 +103,24 @@ export function getInvalidPersistedCronJobReason(
|
||||
}
|
||||
if (scheduleKind === "every") {
|
||||
const everyMs = scheduleRecord.everyMs;
|
||||
if (typeof everyMs !== "number" || !Number.isFinite(everyMs) || everyMs <= 0) {
|
||||
const anchorMs = scheduleRecord.anchorMs;
|
||||
if (
|
||||
asSafeIntegerInRange(everyMs, { min: 1, max: MAX_DATE_TIMESTAMP_MS }) === undefined ||
|
||||
(anchorMs !== undefined &&
|
||||
asSafeIntegerInRange(anchorMs, { min: 0, max: MAX_DATE_TIMESTAMP_MS }) === undefined)
|
||||
) {
|
||||
return "invalid-schedule";
|
||||
}
|
||||
}
|
||||
if (scheduleKind === "cron") {
|
||||
const expr = scheduleRecord.expr;
|
||||
if (typeof expr !== "string" || expr.trim().length === 0) {
|
||||
const staggerMs = scheduleRecord.staggerMs;
|
||||
if (
|
||||
typeof expr !== "string" ||
|
||||
expr.trim().length === 0 ||
|
||||
(staggerMs !== undefined &&
|
||||
asSafeIntegerInRange(staggerMs, { min: 0, max: MAX_DATE_TIMESTAMP_MS }) === undefined)
|
||||
) {
|
||||
return "invalid-schedule";
|
||||
}
|
||||
}
|
||||
@@ -74,7 +138,7 @@ export function getInvalidPersistedCronJobReason(
|
||||
// one such throw would abort the single-pass stream reconcile and block
|
||||
// every valid stream job. Quarantine the row here instead.
|
||||
const batchFieldValid = (value: unknown) =>
|
||||
value === undefined || (typeof value === "number" && Number.isSafeInteger(value));
|
||||
value === undefined || asSafeIntegerInRange(value, {}) !== undefined;
|
||||
if (
|
||||
!Array.isArray(command) ||
|
||||
command.length === 0 ||
|
||||
@@ -123,17 +187,19 @@ export function getInvalidPersistedCronJobReason(
|
||||
) {
|
||||
return "invalid-payload";
|
||||
}
|
||||
if (payloadKind === "systemEvent") {
|
||||
const text = payloadRecord.text;
|
||||
if (typeof text !== "string") {
|
||||
return "invalid-payload";
|
||||
}
|
||||
}
|
||||
if (payloadKind === "agentTurn") {
|
||||
const message = payloadRecord.message;
|
||||
if (typeof message !== "string" || message.trim().length === 0) {
|
||||
return "invalid-payload";
|
||||
}
|
||||
const requiredText =
|
||||
payloadKind === "systemEvent"
|
||||
? payloadRecord.text
|
||||
: payloadKind === "agentTurn"
|
||||
? payloadRecord.message
|
||||
: payloadKind === "script"
|
||||
? payloadRecord.script
|
||||
: undefined;
|
||||
if (
|
||||
(payloadKind === "systemEvent" || payloadKind === "agentTurn" || payloadKind === "script") &&
|
||||
(typeof requiredText !== "string" || (payloadKind !== "systemEvent" && !requiredText.trim()))
|
||||
) {
|
||||
return "invalid-payload";
|
||||
}
|
||||
if (payloadKind === "command") {
|
||||
const argv = payloadRecord.argv;
|
||||
@@ -148,11 +214,5 @@ export function getInvalidPersistedCronJobReason(
|
||||
return "invalid-payload";
|
||||
}
|
||||
}
|
||||
if (payloadKind === "script") {
|
||||
const script = payloadRecord.script;
|
||||
if (typeof script !== "string" || script.trim().length === 0) {
|
||||
return "invalid-payload";
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -4,10 +4,12 @@ import { toPublicCronJob } from "./public-job.js";
|
||||
import type { CronStoredJob } from "./types.js";
|
||||
|
||||
describe("toPublicCronJob", () => {
|
||||
it("strips scheduler-only pacing slots without mutating stored state", () => {
|
||||
it("strips scheduler-only state without mutating the stored job", () => {
|
||||
const job = makeCronJob({
|
||||
state: {
|
||||
nextRunAtMs: 2_000,
|
||||
queuedAtMs: 1_900,
|
||||
startupCatchupAtMs: 2_000,
|
||||
pacedNextRunAtMs: 2_000,
|
||||
forcePreservedNextRunAtMs: 2_000,
|
||||
},
|
||||
@@ -15,8 +17,12 @@ describe("toPublicCronJob", () => {
|
||||
|
||||
const publicJob = toPublicCronJob(job);
|
||||
|
||||
expect(publicJob.state.queuedAtMs).toBeUndefined();
|
||||
expect(publicJob.state.startupCatchupAtMs).toBeUndefined();
|
||||
expect(publicJob.state.pacedNextRunAtMs).toBeUndefined();
|
||||
expect(publicJob.state.forcePreservedNextRunAtMs).toBeUndefined();
|
||||
expect(job.state.queuedAtMs).toBe(1_900);
|
||||
expect(job.state.startupCatchupAtMs).toBe(2_000);
|
||||
expect(job.state.pacedNextRunAtMs).toBe(2_000);
|
||||
expect(job.state.forcePreservedNextRunAtMs).toBe(2_000);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
/** Builds stable identities for cron scheduling inputs. */
|
||||
import {
|
||||
asSafeIntegerInRange,
|
||||
parseStrictFiniteNumber,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { parseCronPacingBounds } from "./pacing.js";
|
||||
import { coerceFiniteScheduleNumber } from "./schedule-number.js";
|
||||
@@ -13,12 +17,16 @@ function readString(record: Record<string, unknown>, key: string): string | unde
|
||||
return normalizeOptionalString(record[key]);
|
||||
}
|
||||
|
||||
function readNumber(record: Record<string, unknown>, key: string): number | undefined {
|
||||
function readScheduleTime(record: Record<string, unknown>, key: string): number | undefined {
|
||||
return coerceFiniteScheduleNumber(record[key]);
|
||||
}
|
||||
|
||||
function readStaggerMs(record: Record<string, unknown>): number | undefined {
|
||||
return normalizeCronStaggerMs(record.staggerMs);
|
||||
function readNumber(record: Record<string, unknown>, key: string): number | undefined {
|
||||
const parsed = parseStrictFiniteNumber(record[key]);
|
||||
return asSafeIntegerInRange(parsed, {
|
||||
min: Number.MIN_SAFE_INTEGER,
|
||||
max: Number.MAX_SAFE_INTEGER,
|
||||
});
|
||||
}
|
||||
|
||||
function schedulePayloadFromRecord(schedule: Record<string, unknown>):
|
||||
@@ -39,10 +47,10 @@ function schedulePayloadFromRecord(schedule: Record<string, unknown>):
|
||||
const rawKind = readString(schedule, "kind")?.toLowerCase();
|
||||
const expr = readString(schedule, "expr");
|
||||
const at = readString(schedule, "at");
|
||||
const everyMs = readNumber(schedule, "everyMs");
|
||||
const anchorMs = readNumber(schedule, "anchorMs");
|
||||
const everyMs = readScheduleTime(schedule, "everyMs");
|
||||
const anchorMs = readScheduleTime(schedule, "anchorMs");
|
||||
const tz = readString(schedule, "tz");
|
||||
const staggerMs = readStaggerMs(schedule);
|
||||
const staggerMs = normalizeCronStaggerMs(schedule.staggerMs);
|
||||
const kind =
|
||||
// Infer legacy shorthand schedule shapes when kind is missing so timer
|
||||
// identity remains stable across old persisted jobs and normalized jobs.
|
||||
@@ -96,15 +104,6 @@ function schedulePayloadFromRecord(schedule: Record<string, unknown>):
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveSchedulePayload(
|
||||
job: CronScheduleIdentityInput,
|
||||
): ReturnType<typeof schedulePayloadFromRecord> {
|
||||
if (job.schedule && typeof job.schedule === "object" && !Array.isArray(job.schedule)) {
|
||||
return schedulePayloadFromRecord(job.schedule as Record<string, unknown>);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolvePacingPayload(
|
||||
job: CronScheduleIdentityInput,
|
||||
): { minMs?: number; maxMs?: number } | null | undefined {
|
||||
@@ -126,7 +125,10 @@ function resolvePacingPayload(
|
||||
|
||||
/** Builds a stable scheduling identity for deciding whether stored timer state is still valid. */
|
||||
export function tryCronScheduleIdentity(job: CronScheduleIdentityInput): string | undefined {
|
||||
const schedule = resolveSchedulePayload(job);
|
||||
const schedule =
|
||||
job.schedule && typeof job.schedule === "object" && !Array.isArray(job.schedule)
|
||||
? schedulePayloadFromRecord(job.schedule as Record<string, unknown>)
|
||||
: undefined;
|
||||
const pacing = resolvePacingPayload(job);
|
||||
if (!schedule || pacing === null) {
|
||||
return undefined;
|
||||
@@ -147,9 +149,5 @@ export function cronSchedulingInputsEqual(
|
||||
): boolean {
|
||||
const previousIdentity = tryCronScheduleIdentity(previous);
|
||||
const nextIdentity = tryCronScheduleIdentity(next);
|
||||
return (
|
||||
previousIdentity !== undefined &&
|
||||
nextIdentity !== undefined &&
|
||||
previousIdentity === nextIdentity
|
||||
);
|
||||
return previousIdentity !== undefined && previousIdentity === nextIdentity;
|
||||
}
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
/** Coerces cron schedule number fields with strict safe-range parsing. */
|
||||
import { parseStrictFiniteNumber } from "@openclaw/normalization-core/number-coercion";
|
||||
/** Coerces cron schedule time fields with strict Date-range parsing. */
|
||||
import {
|
||||
asDateTimestampMs,
|
||||
parseStrictFiniteNumber,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
|
||||
/** Coerces schedule numeric fields without accepting partial, non-finite, or unsafe values. */
|
||||
/** Coerces temporal schedule fields without accepting partial, non-finite, or invalid-Date values. */
|
||||
export function coerceFiniteScheduleNumber(value: unknown): number | undefined {
|
||||
const parsed = parseStrictFiniteNumber(value);
|
||||
return parsed !== undefined && Math.abs(parsed) <= Number.MAX_SAFE_INTEGER ? parsed : undefined;
|
||||
return asDateTimestampMs(parsed);
|
||||
}
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
// Cron schedule tests cover schedule parsing and next-run calculations.
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { Cron } from "croner";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
coerceFiniteScheduleNumber,
|
||||
computeNextRunAtMs,
|
||||
computePreviousRunAtMs,
|
||||
} from "./schedule.js";
|
||||
import { coerceFiniteScheduleNumber } from "./schedule-number.js";
|
||||
import { computeNextRunAtMs, computePreviousRunAtMs } from "./schedule.js";
|
||||
import {
|
||||
clearCronScheduleCacheForTest,
|
||||
getCronScheduleCacheMaxForTest,
|
||||
@@ -385,6 +383,38 @@ describe("cron schedule", () => {
|
||||
expect(next).toBe(anchor + 60_000);
|
||||
});
|
||||
|
||||
it("rejects every schedule numbers outside the ECMAScript Date range", () => {
|
||||
expect(
|
||||
computeNextRunAtMs({ kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS + 1 }, 0),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
computeNextRunAtMs({ kind: "every", everyMs: 1, anchorMs: MAX_DATE_TIMESTAMP_MS + 1 }, 0),
|
||||
).toBeUndefined();
|
||||
expect(computeNextRunAtMs({ kind: "every", everyMs: 0.5, anchorMs: 0 }, 0)).toBeUndefined();
|
||||
expect(computeNextRunAtMs({ kind: "every", everyMs: 1, anchorMs: -1 }, 0)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not return an every occurrence outside the ECMAScript Date range", () => {
|
||||
const anchorMs = MAX_DATE_TIMESTAMP_MS - 1;
|
||||
expect(computeNextRunAtMs({ kind: "every", everyMs: 2, anchorMs }, anchorMs)).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["NaN", Number.NaN],
|
||||
["positive infinity", Number.POSITIVE_INFINITY],
|
||||
["negative infinity", Number.NEGATIVE_INFINITY],
|
||||
["above Date range", MAX_DATE_TIMESTAMP_MS + 1],
|
||||
["below Date range", -MAX_DATE_TIMESTAMP_MS - 1],
|
||||
])("returns undefined instead of throwing for an invalid %s cursor", (_label, nowMs) => {
|
||||
expect(computeNextRunAtMs({ kind: "every", everyMs: 60_000 }, nowMs)).toBeUndefined();
|
||||
expect(
|
||||
computeNextRunAtMs({ kind: "cron", expr: "0 * * * *", tz: "UTC" }, nowMs),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
computePreviousRunAtMs({ kind: "cron", expr: "0 * * * *", tz: "UTC" }, nowMs),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("never returns a past timestamp for Asia/Shanghai daily schedule (#30351)", () => {
|
||||
const nowMs = Date.parse("2026-03-01T00:00:00.000Z");
|
||||
const next = computeNextRunAtMs(
|
||||
@@ -527,6 +557,7 @@ describe("coerceFiniteScheduleNumber", () => {
|
||||
expect(coerceFiniteScheduleNumber("0x10")).toBeUndefined();
|
||||
expect(coerceFiniteScheduleNumber(Number.NaN)).toBeUndefined();
|
||||
expect(coerceFiniteScheduleNumber(Infinity)).toBeUndefined();
|
||||
expect(coerceFiniteScheduleNumber(MAX_DATE_TIMESTAMP_MS + 1)).toBeUndefined();
|
||||
expect(coerceFiniteScheduleNumber(Number.MAX_SAFE_INTEGER + 1)).toBeUndefined();
|
||||
expect(coerceFiniteScheduleNumber(String(Number.MAX_SAFE_INTEGER + 1))).toBeUndefined();
|
||||
expect(coerceFiniteScheduleNumber(null)).toBeUndefined();
|
||||
|
||||
+13
-5
@@ -1,4 +1,5 @@
|
||||
/** Computes at/every/cron schedule timestamps with bounded Croner caching. */
|
||||
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { Cron, CronDate } from "croner";
|
||||
import { parseOffsetlessIsoDateTimeInTimeZone } from "../infra/format-time/parse-offsetless-zoned-datetime.js";
|
||||
@@ -7,8 +8,6 @@ import { parseAbsoluteTimeMs } from "./parse.js";
|
||||
import { coerceFiniteScheduleNumber } from "./schedule-number.js";
|
||||
import type { CronSchedule } from "./types.js";
|
||||
|
||||
export { coerceFiniteScheduleNumber } from "./schedule-number.js";
|
||||
|
||||
const CRON_EVAL_CACHE_MAX = 512;
|
||||
const DAY_MS = 86_400_000;
|
||||
const cronEvalCache = new Map<string, Cron>();
|
||||
@@ -211,6 +210,9 @@ function resolveValidatedNextCronOccurrenceMs(
|
||||
|
||||
/** Computes the next scheduled run timestamp after now for at/every/cron schedules. */
|
||||
export function computeNextRunAtMs(schedule: CronSchedule, nowMs: number): number | undefined {
|
||||
if (asDateTimestampMs(nowMs) === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (schedule.kind === "at") {
|
||||
const atMs = parseAbsoluteTimeMs(schedule.at);
|
||||
if (atMs === null) {
|
||||
@@ -224,15 +226,21 @@ export function computeNextRunAtMs(schedule: CronSchedule, nowMs: number): numbe
|
||||
if (everyMsRaw === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const everyMs = Math.max(1, Math.floor(everyMsRaw));
|
||||
const everyMs = Math.floor(everyMsRaw);
|
||||
if (everyMs < 1) {
|
||||
return undefined;
|
||||
}
|
||||
const anchorRaw = coerceFiniteScheduleNumber(schedule.anchorMs);
|
||||
if (schedule.anchorMs !== undefined && (anchorRaw === undefined || anchorRaw < 0)) {
|
||||
return undefined;
|
||||
}
|
||||
const anchor = Math.max(0, Math.floor(anchorRaw ?? nowMs));
|
||||
if (nowMs < anchor) {
|
||||
return anchor;
|
||||
}
|
||||
const elapsed = nowMs - anchor;
|
||||
const steps = Math.floor(elapsed / everyMs) + 1;
|
||||
return anchor + steps * everyMs;
|
||||
return asDateTimestampMs(anchor + steps * everyMs);
|
||||
}
|
||||
|
||||
if (schedule.kind === "on-exit" || schedule.kind === "stream") {
|
||||
@@ -281,7 +289,7 @@ export function computeNextRunAtMs(schedule: CronSchedule, nowMs: number): numbe
|
||||
|
||||
/** Computes the previous cron-expression run timestamp before now. */
|
||||
export function computePreviousRunAtMs(schedule: CronSchedule, nowMs: number): number | undefined {
|
||||
if (schedule.kind !== "cron") {
|
||||
if (schedule.kind !== "cron" || asDateTimestampMs(nowMs) === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const cron = resolveCronFromSchedule(schedule);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Cron reschedule regression tests cover @schedule jobs after schedule changes.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { computeJobNextRunAtMs } from "./service/jobs.js";
|
||||
import { computeJobNextRunAtMs } from "./service/jobs-scheduling.js";
|
||||
import type { CronJob } from "./types.js";
|
||||
|
||||
const ORIGINAL_AT_MS = Date.parse("2026-02-22T10:00:00.000Z");
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Daily skip regression tests cover missed-run handling for daily cron jobs.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createMockCronStateForJobs } from "./service.test-harness.js";
|
||||
import { recomputeNextRuns, recomputeNextRunsForMaintenance } from "./service/jobs.js";
|
||||
import { recomputeNextRuns, recomputeNextRunsForMaintenance } from "./service/jobs-scheduling.js";
|
||||
import type { CronJob } from "./types.js";
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
// Every-next-run regression tests cover next-run calculations for repeating jobs.
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { computeJobNextRunAtMs } from "./service/jobs.js";
|
||||
import {
|
||||
computeJobNextRunAtMs,
|
||||
hasScheduledNextRunAtMs,
|
||||
resolveJobErrorBackoffUntilMs,
|
||||
} from "./service/jobs-scheduling.js";
|
||||
import type { CronJob } from "./types.js";
|
||||
|
||||
const EVERY_30_MIN_MS = 30 * 60_000;
|
||||
@@ -57,4 +62,38 @@ describe("Cron issue #22895 interval scheduling", () => {
|
||||
const next = computeJobNextRunAtMs(job, nowMs);
|
||||
expect(next).toBe(Date.parse("2026-02-22T10:44:00.000Z"));
|
||||
});
|
||||
|
||||
it("does not return an invalid Date when last-run cadence exceeds the timestamp range", () => {
|
||||
const job = createEveryJob({ lastRunAtMs: MAX_DATE_TIMESTAMP_MS });
|
||||
job.schedule = { kind: "every", everyMs: 1, anchorMs: 0 };
|
||||
|
||||
expect(computeJobNextRunAtMs(job, MAX_DATE_TIMESTAMP_MS - 1)).toBeUndefined();
|
||||
expect(hasScheduledNextRunAtMs(MAX_DATE_TIMESTAMP_MS + 1)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves the inclusive maximum Date timestamp", () => {
|
||||
const job = createEveryJob({ lastRunAtMs: MAX_DATE_TIMESTAMP_MS - 1 });
|
||||
job.schedule = { kind: "every", everyMs: 1, anchorMs: 0 };
|
||||
|
||||
expect(computeJobNextRunAtMs(job, MAX_DATE_TIMESTAMP_MS - 2)).toBe(MAX_DATE_TIMESTAMP_MS);
|
||||
expect(hasScheduledNextRunAtMs(MAX_DATE_TIMESTAMP_MS)).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects malformed intervals, anchors, and overflowing error backoff", () => {
|
||||
const job = createEveryJob({
|
||||
lastRunAtMs: MAX_DATE_TIMESTAMP_MS,
|
||||
lastDurationMs: 1,
|
||||
lastStatus: "error",
|
||||
consecutiveErrors: 1,
|
||||
});
|
||||
|
||||
expect(resolveJobErrorBackoffUntilMs(job)).toBeUndefined();
|
||||
job.state.lastRunAtMs = MAX_DATE_TIMESTAMP_MS - 30_000;
|
||||
job.state.lastDurationMs = 0;
|
||||
expect(resolveJobErrorBackoffUntilMs(job)).toBe(MAX_DATE_TIMESTAMP_MS);
|
||||
job.schedule = { kind: "every", everyMs: 0.5, anchorMs: 0 };
|
||||
expect(computeJobNextRunAtMs(job, ANCHOR_MS)).toBeUndefined();
|
||||
job.schedule = { kind: "every", everyMs: 1, anchorMs: -1 };
|
||||
expect(computeJobNextRunAtMs(job, ANCHOR_MS)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
// Cron service job tests cover job creation, updates, and runtime scheduling.
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
applyDeclarativeJobSpec,
|
||||
applyJobPatch,
|
||||
computeJobNextRunAtMs,
|
||||
computeJobPreviousRunAtOrBeforeMs,
|
||||
createJob,
|
||||
nextWakeAtMs,
|
||||
recomputeNextRuns,
|
||||
recomputeNextRunsForMaintenance,
|
||||
} from "./service/jobs.js";
|
||||
} from "./service/jobs-scheduling.js";
|
||||
import { applyDeclarativeJobSpec, applyJobPatch, createJob } from "./service/jobs.js";
|
||||
import type { CronServiceState } from "./service/state.js";
|
||||
import type { CronJob, CronJobPatch } from "./types.js";
|
||||
|
||||
@@ -558,6 +557,49 @@ function createMockState(
|
||||
} as unknown as CronServiceState;
|
||||
}
|
||||
|
||||
describe("time schedule validation", () => {
|
||||
const now = Date.parse("2026-08-10T00:00:00.000Z");
|
||||
const input = (anchorMs?: number) => ({
|
||||
name: "Date boundary interval",
|
||||
enabled: true,
|
||||
schedule: { kind: "every" as const, everyMs: MAX_DATE_TIMESTAMP_MS, anchorMs },
|
||||
sessionTarget: "main" as const,
|
||||
wakeMode: "now" as const,
|
||||
payload: { kind: "systemEvent" as const, text: "tick" },
|
||||
});
|
||||
|
||||
it("rejects intervals with no representable next run while preserving the inclusive boundary", () => {
|
||||
expect(() => createJob(createMockState(now), input())).toThrow(
|
||||
"cron every schedule has no upcoming run time and would never fire",
|
||||
);
|
||||
expect(createJob(createMockState(now), input(0)).state.nextRunAtMs).toBe(MAX_DATE_TIMESTAMP_MS);
|
||||
});
|
||||
|
||||
it("rejects invalid one-shot timestamps at the service boundary", () => {
|
||||
const maxAt = new Date(MAX_DATE_TIMESTAMP_MS).toISOString();
|
||||
expect(
|
||||
createJob(createMockState(now), {
|
||||
name: "Maximum one-shot",
|
||||
enabled: true,
|
||||
schedule: { kind: "at", at: maxAt },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "now",
|
||||
payload: { kind: "systemEvent", text: "tick" },
|
||||
}).state.nextRunAtMs,
|
||||
).toBe(MAX_DATE_TIMESTAMP_MS);
|
||||
expect(() =>
|
||||
createJob(createMockState(now), {
|
||||
name: "Invalid one-shot",
|
||||
enabled: true,
|
||||
schedule: { kind: "at", at: String(MAX_DATE_TIMESTAMP_MS + 1) },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "now",
|
||||
payload: { kind: "systemEvent", text: "tick" },
|
||||
}),
|
||||
).toThrow("Date-valid absolute timestamp");
|
||||
});
|
||||
});
|
||||
|
||||
describe("announce delivery channel validation", () => {
|
||||
const now = Date.parse("2026-08-02T12:00:00.000Z");
|
||||
const configuredChannels = ["reef", "discord"];
|
||||
@@ -761,6 +803,35 @@ describe("cron tool authority defaults", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("repairs a missing anchor when converging an unchanged every schedule", () => {
|
||||
const createdAtMs = now - 30_000;
|
||||
const job = createJob(createMockState(now), {
|
||||
name: "legacy declaration",
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "now",
|
||||
payload: { kind: "systemEvent", text: "tick" },
|
||||
});
|
||||
job.createdAtMs = createdAtMs;
|
||||
job.schedule = { kind: "every", everyMs: 60_000 };
|
||||
|
||||
applyDeclarativeJobSpec(
|
||||
job,
|
||||
{
|
||||
name: job.name,
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "now",
|
||||
payload: { kind: "systemEvent", text: "tick" },
|
||||
},
|
||||
{ enabledExplicit: false, nowMs: now },
|
||||
);
|
||||
|
||||
expect(job.schedule).toEqual({ kind: "every", everyMs: 60_000, anchorMs: createdAtMs });
|
||||
});
|
||||
|
||||
it("adopts explicit authority when a declaration becomes tool-bearing", () => {
|
||||
const job: CronJob = {
|
||||
id: "declared-trigger",
|
||||
@@ -1242,6 +1313,38 @@ describe("cron stagger defaults", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves staggering when declarative convergence keeps the cron expression", () => {
|
||||
const now = Date.now();
|
||||
const job = createJob(createMockState(now), {
|
||||
name: "declared hourly",
|
||||
enabled: true,
|
||||
schedule: { kind: "cron", expr: "0 * * * *", tz: "UTC", staggerMs: 120_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "now",
|
||||
payload: { kind: "systemEvent", text: "tick" },
|
||||
});
|
||||
|
||||
applyDeclarativeJobSpec(
|
||||
job,
|
||||
{
|
||||
name: job.name,
|
||||
enabled: true,
|
||||
schedule: { kind: "cron", expr: "0 * * * *", tz: "America/Los_Angeles" },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "now",
|
||||
payload: { kind: "systemEvent", text: "tick" },
|
||||
},
|
||||
{ enabledExplicit: false, nowMs: now },
|
||||
);
|
||||
|
||||
expect(job.schedule).toEqual({
|
||||
kind: "cron",
|
||||
expr: "0 * * * *",
|
||||
tz: "America/Los_Angeles",
|
||||
staggerMs: 120_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("applies default stagger when switching from every to top-of-hour cron", () => {
|
||||
const now = Date.now();
|
||||
const job: CronJob = {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Top-of-hour stagger tests cover spreading jobs that would otherwise collide.
|
||||
import crypto from "node:crypto";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { computeJobNextRunAtMs } from "./service/jobs.js";
|
||||
import { computeJobNextRunAtMs } from "./service/jobs-scheduling.js";
|
||||
import type { CronJob } from "./types.js";
|
||||
|
||||
const DEFAULT_TOP_OF_HOUR_STAGGER_MS = 5 * 60 * 1000;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Cron service regression tests cover historical scheduling edge cases.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createMockCronStateForJobs } from "./service.test-harness.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./service/jobs.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./service/jobs-scheduling.js";
|
||||
import { reserveQueuedCronRun } from "./service/run-admission.js";
|
||||
import type { CronJob } from "./types.js";
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { setupCronServiceSuite } from "./service.test-harness.js";
|
||||
import { start } from "./service/ops-lifecycle.js";
|
||||
import { status } from "./service/ops-read.js";
|
||||
import { createCronServiceState } from "./service/state.js";
|
||||
import { runMissedJobs } from "./service/timer.js";
|
||||
import { onTimer } from "./service/timer.test-support.js";
|
||||
import { saveCronStore } from "./store.js";
|
||||
import * as cronStoreModule from "./store.js";
|
||||
import { loadCronStore, saveCronStore } from "./store.js";
|
||||
import type { CronJob } from "./types.js";
|
||||
|
||||
const { logger: noopLogger, makeStorePath } = setupCronServiceSuite({
|
||||
@@ -13,6 +16,21 @@ const { logger: noopLogger, makeStorePath } = setupCronServiceSuite({
|
||||
});
|
||||
|
||||
describe("CronService startup catch-up repair scoping", () => {
|
||||
function createDateBoundaryEveryJob(id: string, nextRunAtMs: number): CronJob {
|
||||
return {
|
||||
id,
|
||||
name: `job-${id}`,
|
||||
enabled: true,
|
||||
createdAtMs: nextRunAtMs - 60_000,
|
||||
updatedAtMs: nextRunAtMs - 60_000,
|
||||
schedule: { kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS, anchorMs: 0 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: `tick-${id}` },
|
||||
state: { nextRunAtMs },
|
||||
};
|
||||
}
|
||||
|
||||
function createHourlyCronJob(id: string, nextRunAtMs: number): CronJob {
|
||||
return {
|
||||
id,
|
||||
@@ -143,4 +161,92 @@ describe("CronService startup catch-up repair scoping", () => {
|
||||
state.stopped = true;
|
||||
await store.cleanup();
|
||||
});
|
||||
|
||||
it("disables startup catch-up deferrals that exceed the Date range", async () => {
|
||||
const store = await makeStorePath();
|
||||
const now = MAX_DATE_TIMESTAMP_MS - 2_000;
|
||||
await saveCronStore(store.storePath, {
|
||||
version: 1,
|
||||
jobs: [
|
||||
createDateBoundaryEveryJob("date-limit-0", now - 60_000),
|
||||
createDateBoundaryEveryJob("date-limit-1", now - 50_000),
|
||||
createDateBoundaryEveryJob("date-limit-2", now - 40_000),
|
||||
],
|
||||
});
|
||||
const order: string[] = [];
|
||||
const deferredAutoDisableReasons = new Set([
|
||||
"cron:date-limit-1:auto-disabled",
|
||||
"cron:date-limit-2:auto-disabled",
|
||||
]);
|
||||
const enqueueSystemEvent = vi.fn((_text: string, context?: { contextKey?: string }) => {
|
||||
if (context?.contextKey && deferredAutoDisableReasons.has(context.contextKey)) {
|
||||
order.push("notify");
|
||||
}
|
||||
});
|
||||
const requestHeartbeat = vi.fn((request: { reason?: string }) => {
|
||||
if (request.reason && deferredAutoDisableReasons.has(request.reason)) {
|
||||
order.push("heartbeat");
|
||||
}
|
||||
});
|
||||
const state = createCronServiceState({
|
||||
cronEnabled: true,
|
||||
storePath: store.storePath,
|
||||
log: noopLogger,
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent,
|
||||
requestHeartbeat,
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
maxMissedJobsPerRestart: 1,
|
||||
missedJobStaggerMs: 5_000,
|
||||
});
|
||||
const save = cronStoreModule.saveCronJobsStore;
|
||||
const saveSpy = vi
|
||||
.spyOn(cronStoreModule, "saveCronJobsStore")
|
||||
.mockImplementation(async (...args) => {
|
||||
if (
|
||||
args[1].jobs.some(
|
||||
(job) => job.id !== "date-limit-0" && job.state.autoDisabled !== undefined,
|
||||
)
|
||||
) {
|
||||
expect(order).toEqual([]);
|
||||
const result = await save(...args);
|
||||
expect(order).toEqual([]);
|
||||
order.push("persist");
|
||||
return result;
|
||||
}
|
||||
return await save(...args);
|
||||
});
|
||||
|
||||
try {
|
||||
await runMissedJobs(state);
|
||||
|
||||
const deferred = (state.store?.jobs ?? []).filter((job) => job.id !== "date-limit-0");
|
||||
expect(deferred).toHaveLength(2);
|
||||
for (const job of deferred) {
|
||||
expect(job.enabled).toBe(false);
|
||||
expect(job.state.nextRunAtMs).toBeUndefined();
|
||||
expect(job.state.startupCatchupAtMs).toBeUndefined();
|
||||
expect(job.state.autoDisabled).toEqual({
|
||||
reason: "schedule-errors",
|
||||
atMs: now,
|
||||
consecutiveErrors: 1,
|
||||
});
|
||||
}
|
||||
expect(order).toEqual(["persist", "notify", "heartbeat", "notify", "heartbeat"]);
|
||||
expect((await loadCronStore(store.storePath)).jobs).toEqual(
|
||||
expect.arrayContaining(
|
||||
deferred.map((job) =>
|
||||
expect.objectContaining({
|
||||
id: job.id,
|
||||
enabled: false,
|
||||
state: expect.objectContaining({ autoDisabled: job.state.autoDisabled }),
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
saveSpy.mockRestore();
|
||||
await store.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { CronService } from "./service.js";
|
||||
import { setupCronServiceSuite } from "./service.test-harness.js";
|
||||
@@ -177,6 +178,22 @@ describe("cron stream schedule validation", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects invalid scheduler timestamps from external event sources", async () => {
|
||||
const cron = await createCron(true);
|
||||
try {
|
||||
const created = await cron.add(streamJob());
|
||||
|
||||
await expect(
|
||||
cron.recordExternalFailure(created.id, "invalid source state", {
|
||||
startupCatchupAtMs: MAX_DATE_TIMESTAMP_MS + 1,
|
||||
}),
|
||||
).rejects.toThrow("cron state.startupCatchupAtMs");
|
||||
expect(cron.getJob(created.id)?.state.startupCatchupAtMs).toBeUndefined();
|
||||
} finally {
|
||||
cron.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it("rotates logical source identity only when source ownership changes", async () => {
|
||||
const cron = await createCron(true);
|
||||
try {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { createDeferred } from "../../test/helpers/promise.js";
|
||||
import type { CronEvent } from "./service.js";
|
||||
import { CronService } from "./service.js";
|
||||
import { setupCronServiceSuite } from "./service.test-harness.js";
|
||||
import { computeJobNextRunAtMs } from "./service/jobs.js";
|
||||
import { computeJobNextRunAtMs } from "./service/jobs-scheduling.js";
|
||||
import type { CronServiceDeps } from "./service/state.js";
|
||||
import { loadCronStore } from "./store.js";
|
||||
import { cronStoreKey } from "./store/key.js";
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
/** Scheduling state and next-run computation for cron jobs. */
|
||||
import crypto from "node:crypto";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
|
||||
import { pruneMapToMaxSize } from "../../infra/map-size.js";
|
||||
import { isCronJobActive } from "../active-jobs.js";
|
||||
import { parseAbsoluteTimeMs } from "../parse.js";
|
||||
import {
|
||||
coerceFiniteScheduleNumber,
|
||||
computeNextRunAtMs,
|
||||
computePreviousRunAtMs,
|
||||
} from "../schedule.js";
|
||||
import { coerceFiniteScheduleNumber } from "../schedule-number.js";
|
||||
import { computeNextRunAtMs, computePreviousRunAtMs } from "../schedule.js";
|
||||
import { resolveCronStaggerMs } from "../stagger.js";
|
||||
import { createCronStreamSourceIdentity, resolveCronStreamBatching } from "../stream-schedule.js";
|
||||
import type { CronJob, CronSchedule } from "../types.js";
|
||||
@@ -56,7 +54,7 @@ export const DEFAULT_ERROR_BACKOFF_SCHEDULE_MS = [
|
||||
];
|
||||
|
||||
function isFiniteTimestamp(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
return asDateTimestampMs(value) !== undefined;
|
||||
}
|
||||
|
||||
/** Returns whether a stored next-run timestamp is finite and schedulable. */
|
||||
@@ -99,7 +97,7 @@ export function resolveJobErrorBackoffUntilMs(
|
||||
? Math.max(0, Math.floor(job.state.lastDurationMs))
|
||||
: 0;
|
||||
const lastEndedAtMs = job.state.lastRunAtMs + lastDurationMs;
|
||||
return lastEndedAtMs + errorBackoffMs(consecutiveErrors, scheduleMs);
|
||||
return asDateTimestampMs(lastEndedAtMs + errorBackoffMs(consecutiveErrors, scheduleMs));
|
||||
}
|
||||
|
||||
function resolveStableCronOffsetMs(jobId: string, staggerMs: number) {
|
||||
@@ -139,7 +137,7 @@ function computeStaggeredCronNextRunAtMs(job: CronJob, nowMs: number) {
|
||||
return undefined;
|
||||
}
|
||||
const shifted = baseNext + offsetMs;
|
||||
if (shifted > nowMs) {
|
||||
if (isFiniteTimestamp(shifted) && shifted > nowMs) {
|
||||
return shifted;
|
||||
}
|
||||
cursorMs = Math.max(cursorMs + 1, baseNext + 1_000);
|
||||
@@ -167,7 +165,7 @@ function computeStaggeredCronPreviousRunAtMs(job: CronJob, nowMs: number) {
|
||||
return undefined;
|
||||
}
|
||||
const shifted = basePrevious + offsetMs;
|
||||
if (shifted <= nowMs) {
|
||||
if (isFiniteTimestamp(shifted) && shifted <= nowMs) {
|
||||
return shifted;
|
||||
}
|
||||
cursorMs = Math.max(0, basePrevious - 1_000);
|
||||
@@ -178,7 +176,7 @@ function computeStaggeredCronPreviousRunAtMs(job: CronJob, nowMs: number) {
|
||||
function computeStaggeredCronPreviousRunAtOrBeforeMs(job: CronJob, nowMs: number) {
|
||||
const previous = computeStaggeredCronPreviousRunAtMs(job, nowMs);
|
||||
const probeMs = nowMs + 1_000;
|
||||
if (!Number.isFinite(probeMs)) {
|
||||
if (!isFiniteTimestamp(probeMs)) {
|
||||
return previous;
|
||||
}
|
||||
|
||||
@@ -205,7 +203,6 @@ function isStaggeredCronRunAtMs(job: CronJob, runAtMs: number): boolean {
|
||||
}
|
||||
|
||||
function isPendingErrorBackoffSlot(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
nextRunAtMs: number;
|
||||
nowMs: number;
|
||||
@@ -215,12 +212,8 @@ function isPendingErrorBackoffSlot(params: {
|
||||
return backoffUntilMs !== undefined && nowMs < backoffUntilMs && nextRunAtMs <= backoffUntilMs;
|
||||
}
|
||||
|
||||
function shouldRepairFutureCronNextRunAtMs(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
nowMs: number;
|
||||
}): boolean {
|
||||
const { state, job, nowMs } = params;
|
||||
function shouldRepairFutureCronNextRunAtMs(params: { job: CronJob; nowMs: number }): boolean {
|
||||
const { job, nowMs } = params;
|
||||
const nextRun = job.state.nextRunAtMs;
|
||||
if (
|
||||
job.schedule.kind !== "cron" ||
|
||||
@@ -235,7 +228,7 @@ function shouldRepairFutureCronNextRunAtMs(params: {
|
||||
// Error retries may intentionally use a non-cron future timestamp while
|
||||
// backoff is pending. Once the retry window has elapsed, stale future cron
|
||||
// slots should be eligible for the same repair as ordinary schedule state.
|
||||
if (isPendingErrorBackoffSlot({ state, job, nextRunAtMs: nextRun, nowMs })) {
|
||||
if (isPendingErrorBackoffSlot({ job, nextRunAtMs: nextRun, nowMs })) {
|
||||
return false;
|
||||
}
|
||||
let naturalNext: number | undefined;
|
||||
@@ -290,6 +283,14 @@ export function resolveEveryAnchorMs(params: {
|
||||
return 0;
|
||||
}
|
||||
|
||||
function hasInvalidExplicitEveryAnchor(schedule: { anchorMs?: number }): boolean {
|
||||
if (schedule.anchorMs === undefined) {
|
||||
return false;
|
||||
}
|
||||
const coerced = coerceFiniteScheduleNumber(schedule.anchorMs);
|
||||
return coerced === undefined || coerced < 0;
|
||||
}
|
||||
|
||||
/** Finds an in-memory cron job or throws the public unknown-id error. */
|
||||
export function findJobOrThrow(state: CronServiceState, id: string) {
|
||||
const job = state.store?.jobs.find((j) => j.id === id);
|
||||
@@ -310,14 +311,23 @@ export function computeJobNextRunAtMs(job: CronJob, nowMs: number): number | und
|
||||
return undefined;
|
||||
}
|
||||
if (job.schedule.kind === "every") {
|
||||
if (hasInvalidExplicitEveryAnchor(job.schedule)) {
|
||||
return undefined;
|
||||
}
|
||||
const everyMsRaw = coerceFiniteScheduleNumber(job.schedule.everyMs);
|
||||
if (everyMsRaw === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const everyMs = Math.max(1, Math.floor(everyMsRaw));
|
||||
const everyMs = Math.floor(everyMsRaw);
|
||||
if (everyMs < 1) {
|
||||
return undefined;
|
||||
}
|
||||
const lastRunAtMs = job.state.lastRunAtMs;
|
||||
if (typeof lastRunAtMs === "number" && Number.isFinite(lastRunAtMs)) {
|
||||
if (isFiniteTimestamp(lastRunAtMs)) {
|
||||
const nextFromLastRun = Math.floor(lastRunAtMs) + everyMs;
|
||||
if (!isFiniteTimestamp(nextFromLastRun)) {
|
||||
return undefined;
|
||||
}
|
||||
if (nextFromLastRun > nowMs) {
|
||||
return nextFromLastRun;
|
||||
}
|
||||
@@ -422,7 +432,7 @@ function normalizeJobTickState(params: { state: CronServiceState; job: CronJob;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if (job.schedule.kind === "every") {
|
||||
if (job.schedule.kind === "every" && !hasInvalidExplicitEveryAnchor(job.schedule)) {
|
||||
const normalizedAnchorMs = resolveEveryAnchorMs({
|
||||
schedule: job.schedule,
|
||||
fallbackAnchorMs: isFiniteTimestamp(job.createdAtMs) ? job.createdAtMs : nowMs,
|
||||
@@ -607,7 +617,7 @@ export function recomputeNextRuns(state: CronServiceState): boolean {
|
||||
const isDueOrMissing = !hasScheduledNextRunAtMs(nextRun) || now >= nextRun;
|
||||
return (
|
||||
!hasForcePreservedNextRun &&
|
||||
(isDueOrMissing || shouldRepairFutureCronNextRunAtMs({ state, job, nowMs: now })) &&
|
||||
(isDueOrMissing || shouldRepairFutureCronNextRunAtMs({ job, nowMs: now })) &&
|
||||
recomputeJobNextRunAtMs({ state, job, nowMs: now })
|
||||
);
|
||||
});
|
||||
@@ -681,7 +691,7 @@ export function recomputeNextRunsForMaintenance(
|
||||
!hasPendingStartupCatchup &&
|
||||
!hasPendingPacedNextRun &&
|
||||
!hasForcePreservedNextRun &&
|
||||
shouldRepairFutureCronNextRunAtMs({ state, job, nowMs: now })
|
||||
shouldRepairFutureCronNextRunAtMs({ job, nowMs: now })
|
||||
) {
|
||||
changed = recomputeJob(job, now) || changed;
|
||||
} else if (
|
||||
|
||||
@@ -6,6 +6,7 @@ import { normalizeAgentId } from "../../routing/session-key.js";
|
||||
import { compileSafeRegexDetailed } from "../../security/safe-regex.js";
|
||||
import { resolveCronDeliveryPlan } from "../delivery-plan.js";
|
||||
import { parseCronPacingBounds } from "../pacing.js";
|
||||
import { parseAbsoluteTimeMs } from "../parse.js";
|
||||
import { assertSafeCronSessionTargetId } from "../session-target.js";
|
||||
import type { CronDelivery, CronJob, CronJobPatch } from "../types.js";
|
||||
import { normalizeHttpWebhookUrl } from "../webhook-url.js";
|
||||
@@ -153,17 +154,26 @@ export function assertStreamScheduleSupport(
|
||||
}
|
||||
}
|
||||
|
||||
export function assertCronExpressionSatisfiable(
|
||||
export function assertTimeScheduleSatisfiable(
|
||||
job: CronJob,
|
||||
nowMs: number,
|
||||
computeJobNextRunAtMs: (job: CronJob, nowMs: number) => number | undefined,
|
||||
) {
|
||||
if (job.schedule.kind !== "cron") {
|
||||
if (job.schedule.kind === "at") {
|
||||
if (parseAbsoluteTimeMs(job.schedule.at) === null) {
|
||||
throw new Error("cron at schedule must contain a Date-valid absolute timestamp");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (job.schedule.kind !== "cron" && job.schedule.kind !== "every") {
|
||||
return;
|
||||
}
|
||||
if (computeJobNextRunAtMs({ ...job, enabled: true }, nowMs) !== undefined) {
|
||||
return;
|
||||
}
|
||||
if (job.schedule.kind === "every") {
|
||||
throw new Error("cron every schedule has no upcoming run time and would never fire");
|
||||
}
|
||||
throw new Error(
|
||||
`cron expression "${job.schedule.expr}" has no upcoming run time and would never fire`,
|
||||
);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Cron job patch tests cover applying partial updates to scheduled jobs.
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveCronDeliveryPlan, resolveFailureDestination } from "../delivery-plan.js";
|
||||
import { projectCronJobThroughStorageCodec } from "../store/row-codec.js";
|
||||
@@ -125,6 +126,19 @@ describe("schedule activation ownership", () => {
|
||||
|
||||
expect(job.state.scheduleActivatedAtMs).toBe(456);
|
||||
});
|
||||
|
||||
it.each(["nextRunAtMs", "startupCatchupAtMs", "pacedNextRunAtMs"] as const)(
|
||||
"rejects out-of-Date-range caller state for %s",
|
||||
(field) => {
|
||||
const job = makeJob({ enabled: false });
|
||||
const patch = {
|
||||
state: { [field]: MAX_DATE_TIMESTAMP_MS + 1 },
|
||||
} as CronJobPatch;
|
||||
|
||||
expect(() => applyJobPatch(job, patch)).toThrow(`cron state.${field}`);
|
||||
expect(job.state[field]).toBeUndefined();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("applyJobPatch delivery merge", () => {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Schedule error isolation tests cover one bad job not blocking other cron jobs.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CronJob, CronStoreFile } from "../types.js";
|
||||
import { recomputeNextRuns } from "./jobs.js";
|
||||
import { recomputeNextRuns } from "./jobs-scheduling.js";
|
||||
import type { CronServiceState } from "./state.js";
|
||||
|
||||
function createMockState(jobs: CronJob[]): CronServiceState {
|
||||
|
||||
+194
-185
@@ -7,6 +7,7 @@ import {
|
||||
import type { CronConfig } from "../../config/types.cron.js";
|
||||
import { normalizeOptionalAccountId } from "../../routing/account-id.js";
|
||||
import { resolveCronDeliveryPlan } from "../delivery-plan.js";
|
||||
import { assertCronJobStateTimestamps } from "../persisted-shape.js";
|
||||
import type { CronScheduledToolPolicy } from "../scheduled-tool-policy.js";
|
||||
import { normalizeCronScriptPayload } from "../script-payload.js";
|
||||
import { normalizeCronStaggerMs, resolveDefaultCronStaggerMs } from "../stagger.js";
|
||||
@@ -20,6 +21,7 @@ import type {
|
||||
CronJobCreate,
|
||||
CronJobPatch,
|
||||
CronJobState,
|
||||
CronSchedule,
|
||||
CronStoredJob,
|
||||
CronToolsAllowProvenance,
|
||||
} from "../types.js";
|
||||
@@ -36,7 +38,7 @@ import {
|
||||
} from "./jobs-tool-policy.js";
|
||||
import {
|
||||
assertAnnounceDeliveryChannelSupport,
|
||||
assertCronExpressionSatisfiable,
|
||||
assertTimeScheduleSatisfiable,
|
||||
assertDeliverySupport,
|
||||
assertFailureDestinationSupport,
|
||||
assertMainSessionAgentId,
|
||||
@@ -54,26 +56,151 @@ import type { CronServiceState } from "./state.js";
|
||||
const CRON_DECLARATIVE_LABEL_MAX_LENGTH = 200;
|
||||
type DeliveryValidationOptions = { configuredChannels?: readonly string[] };
|
||||
|
||||
export { assertSupportedJobSpec };
|
||||
type ScheduleNormalizationContext =
|
||||
| { kind: "create"; nowMs: number }
|
||||
| { kind: "patch"; previous: CronSchedule }
|
||||
| { kind: "declarative"; previous: CronSchedule; nowMs: number; fallbackAnchorMs: number };
|
||||
|
||||
export {
|
||||
DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
|
||||
hasScheduledNextRunAtMs,
|
||||
resolveJobLastRunStatus,
|
||||
errorBackoffMs,
|
||||
resolveJobErrorBackoffUntilMs,
|
||||
findJobOrThrow,
|
||||
isJobEnabled,
|
||||
computeJobNextRunAtMs,
|
||||
computeJobPreviousRunAtOrBeforeMs,
|
||||
recordScheduleComputeError,
|
||||
recomputeNextRuns,
|
||||
recomputeNextRunsForMaintenance,
|
||||
nextWakeAtMs,
|
||||
hasActiveCronRun,
|
||||
isJobDue,
|
||||
resolveJobPayloadTextForMain,
|
||||
} from "./jobs-scheduling.js";
|
||||
function normalizeJobSchedule(
|
||||
schedule: CronSchedule,
|
||||
context: ScheduleNormalizationContext,
|
||||
): CronSchedule {
|
||||
if (schedule.kind === "every") {
|
||||
if (context.kind === "patch") {
|
||||
return schedule;
|
||||
}
|
||||
if (context.kind === "create") {
|
||||
return {
|
||||
...schedule,
|
||||
anchorMs: resolveEveryAnchorMs({ schedule, fallbackAnchorMs: context.nowMs }),
|
||||
};
|
||||
}
|
||||
if (schedule.anchorMs !== undefined) {
|
||||
return schedule;
|
||||
}
|
||||
const anchorMs =
|
||||
context.previous.kind === "every" && context.previous.everyMs === schedule.everyMs
|
||||
? resolveEveryAnchorMs({
|
||||
schedule: context.previous,
|
||||
fallbackAnchorMs: context.fallbackAnchorMs,
|
||||
})
|
||||
: context.nowMs;
|
||||
return {
|
||||
...schedule,
|
||||
anchorMs,
|
||||
};
|
||||
}
|
||||
if (schedule.kind === "cron") {
|
||||
const explicitStaggerMs = normalizeCronStaggerMs(schedule.staggerMs);
|
||||
if (explicitStaggerMs !== undefined) {
|
||||
return { ...schedule, staggerMs: explicitStaggerMs };
|
||||
}
|
||||
if (
|
||||
context.kind !== "create" &&
|
||||
context.previous.kind === "cron" &&
|
||||
context.previous.expr === schedule.expr
|
||||
) {
|
||||
return { ...schedule, staggerMs: context.previous.staggerMs };
|
||||
}
|
||||
const defaultStaggerMs = resolveDefaultCronStaggerMs(schedule.expr);
|
||||
if (defaultStaggerMs !== undefined) {
|
||||
return { ...schedule, staggerMs: defaultStaggerMs };
|
||||
}
|
||||
return context.kind === "declarative" ? { ...schedule } : schedule;
|
||||
}
|
||||
const input = context.kind === "declarative" ? structuredClone(schedule) : schedule;
|
||||
return normalizeStreamScheduleBounds(input);
|
||||
}
|
||||
|
||||
function normalizeDeclarativeLabel(
|
||||
value: unknown,
|
||||
field: "declarationKey" | "displayName",
|
||||
nullable = false,
|
||||
): string | undefined {
|
||||
const normalized = normalizeOptionalString(value);
|
||||
if (!(nullable && value == null) && value !== undefined && !normalized) {
|
||||
throw new Error(`cron ${field} must not be blank`);
|
||||
}
|
||||
if (normalized && normalized.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
|
||||
throw new Error(
|
||||
`cron ${field} must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
type JobValidationContext =
|
||||
| { kind: "create"; cronConfig?: CronConfig; defaultAgentId?: string; nowMs: number }
|
||||
| {
|
||||
kind: "patch";
|
||||
patch: CronJobPatch;
|
||||
defaultAgentId?: string;
|
||||
nowMs?: number;
|
||||
cronConfig?: CronConfig;
|
||||
}
|
||||
| {
|
||||
kind: "declarative";
|
||||
input: CronJobCreate;
|
||||
defaultAgentId?: string;
|
||||
nowMs: number;
|
||||
cronConfig?: CronConfig;
|
||||
};
|
||||
|
||||
function validateFullJob(
|
||||
job: CronStoredJob,
|
||||
context: JobValidationContext,
|
||||
configuredChannels?: readonly string[],
|
||||
) {
|
||||
const cronConfig = context.cronConfig;
|
||||
const triggerTouched =
|
||||
context.kind === "create"
|
||||
? job.trigger !== undefined
|
||||
: context.kind === "patch"
|
||||
? context.patch.trigger != null
|
||||
: context.input.trigger !== undefined;
|
||||
const scriptTouched =
|
||||
context.kind === "create"
|
||||
? job.payload.kind === "script"
|
||||
: context.kind === "patch"
|
||||
? context.patch.payload?.kind === "script"
|
||||
: context.input.payload.kind === "script";
|
||||
const streamTouched =
|
||||
context.kind !== "patch" ||
|
||||
context.patch.enabled === true ||
|
||||
context.patch.schedule?.kind === "stream";
|
||||
const validateCapabilities = () => {
|
||||
assertTriggerSupport(job, { cronConfig, requireEnabled: triggerTouched });
|
||||
assertScriptPayloadSupport(job, {
|
||||
cronConfig,
|
||||
requireEnabled: scriptTouched,
|
||||
...(context.kind === "patch" ? { validateSyntax: context.patch.payload !== undefined } : {}),
|
||||
});
|
||||
assertStreamScheduleSupport(job, { cronConfig, requireEnabled: streamTouched });
|
||||
};
|
||||
if (context.kind === "declarative") {
|
||||
validateCapabilities();
|
||||
}
|
||||
assertSupportedJobSpec(job);
|
||||
assertPacingSupport(job);
|
||||
if (context.kind !== "declarative") {
|
||||
validateCapabilities();
|
||||
}
|
||||
assertMainSessionAgentId(job, context.defaultAgentId);
|
||||
assertDeliverySupport(job);
|
||||
assertAnnounceDeliveryChannelSupport(
|
||||
job,
|
||||
configuredChannels,
|
||||
context.kind === "patch" ? context.patch : undefined,
|
||||
);
|
||||
assertFailureDestinationSupport(job);
|
||||
const scheduleTouched =
|
||||
context.kind !== "patch" ||
|
||||
context.patch.schedule !== undefined ||
|
||||
context.patch.enabled === true;
|
||||
if (context.nowMs !== undefined && scheduleTouched) {
|
||||
assertTimeScheduleSatisfiable(job, context.nowMs, computeJobNextRunAtMs);
|
||||
}
|
||||
}
|
||||
/** Creates a normalized cron job row from public add input and computes its initial schedule. */
|
||||
export function createJob(
|
||||
state: CronServiceState,
|
||||
@@ -85,27 +212,7 @@ export function createJob(
|
||||
): CronStoredJob {
|
||||
const now = state.deps.nowMs();
|
||||
const id = normalizeOptionalString(input.id) ?? crypto.randomUUID();
|
||||
const schedule =
|
||||
input.schedule.kind === "every"
|
||||
? {
|
||||
...input.schedule,
|
||||
anchorMs: resolveEveryAnchorMs({
|
||||
schedule: input.schedule,
|
||||
fallbackAnchorMs: now,
|
||||
}),
|
||||
}
|
||||
: input.schedule.kind === "cron"
|
||||
? (() => {
|
||||
const explicitStaggerMs = normalizeCronStaggerMs(input.schedule.staggerMs);
|
||||
if (explicitStaggerMs !== undefined) {
|
||||
return { ...input.schedule, staggerMs: explicitStaggerMs };
|
||||
}
|
||||
const defaultStaggerMs = resolveDefaultCronStaggerMs(input.schedule.expr);
|
||||
return defaultStaggerMs !== undefined
|
||||
? { ...input.schedule, staggerMs: defaultStaggerMs }
|
||||
: input.schedule;
|
||||
})()
|
||||
: normalizeStreamScheduleBounds(input.schedule);
|
||||
const schedule = normalizeJobSchedule(input.schedule, { kind: "create", nowMs: now });
|
||||
const deleteAfterRun =
|
||||
typeof input.deleteAfterRun === "boolean"
|
||||
? input.deleteAfterRun
|
||||
@@ -113,24 +220,8 @@ export function createJob(
|
||||
? true
|
||||
: undefined;
|
||||
const enabled = typeof input.enabled === "boolean" ? input.enabled : true;
|
||||
const declarationKey = normalizeOptionalString(input.declarationKey);
|
||||
if (input.declarationKey !== undefined && !declarationKey) {
|
||||
throw new Error("cron declarationKey must not be blank");
|
||||
}
|
||||
if (declarationKey && declarationKey.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
|
||||
throw new Error(
|
||||
`cron declarationKey must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
|
||||
);
|
||||
}
|
||||
const displayName = normalizeOptionalString(input.displayName);
|
||||
if (input.displayName !== undefined && !displayName) {
|
||||
throw new Error("cron displayName must not be blank");
|
||||
}
|
||||
if (displayName && displayName.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
|
||||
throw new Error(
|
||||
`cron displayName must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
|
||||
);
|
||||
}
|
||||
const declarationKey = normalizeDeclarativeLabel(input.declarationKey, "declarationKey");
|
||||
const displayName = normalizeDeclarativeLabel(input.displayName, "displayName");
|
||||
const ownerAgentId = normalizeOptionalAgentId(input.owner?.agentId);
|
||||
const ownerSessionKey = normalizeOptionalString(input.owner?.sessionKey);
|
||||
const ownerAccountId = normalizeOptionalAccountId(input.owner?.accountId);
|
||||
@@ -138,6 +229,8 @@ export function createJob(
|
||||
// Schedule activation is stamped only by committed scheduling mutations.
|
||||
// Accepting caller state here would let imports spoof restart catch-up ownership.
|
||||
delete initialState.scheduleActivatedAtMs;
|
||||
delete initialState.autoDisabled;
|
||||
assertCronJobStateTimestamps(initialState);
|
||||
const job: CronStoredJob = {
|
||||
id,
|
||||
...(declarationKey ? { declarationKey } : {}),
|
||||
@@ -186,25 +279,16 @@ export function createJob(
|
||||
explicitlyMutatesToolsAllow: true,
|
||||
toolsAllowProvenance: opts?.toolsAllowProvenance,
|
||||
});
|
||||
assertSupportedJobSpec(job);
|
||||
assertPacingSupport(job);
|
||||
assertTriggerSupport(job, {
|
||||
cronConfig: state.deps.cronConfig,
|
||||
requireEnabled: job.trigger !== undefined,
|
||||
});
|
||||
assertScriptPayloadSupport(job, {
|
||||
cronConfig: state.deps.cronConfig,
|
||||
requireEnabled: job.payload.kind === "script",
|
||||
});
|
||||
assertStreamScheduleSupport(job, {
|
||||
cronConfig: state.deps.cronConfig,
|
||||
requireEnabled: true,
|
||||
});
|
||||
assertMainSessionAgentId(job, state.deps.defaultAgentId);
|
||||
assertDeliverySupport(job);
|
||||
assertAnnounceDeliveryChannelSupport(job, opts?.configuredChannels);
|
||||
assertFailureDestinationSupport(job);
|
||||
assertCronExpressionSatisfiable(job, now, computeJobNextRunAtMs);
|
||||
validateFullJob(
|
||||
job,
|
||||
{
|
||||
kind: "create",
|
||||
cronConfig: state.deps.cronConfig,
|
||||
defaultAgentId: state.deps.defaultAgentId,
|
||||
nowMs: now,
|
||||
},
|
||||
opts?.configuredChannels,
|
||||
);
|
||||
job.state.nextRunAtMs = computeJobNextRunAtMs(job, now);
|
||||
return job;
|
||||
}
|
||||
@@ -231,15 +315,7 @@ export function applyJobPatch(
|
||||
job.description = normalizeOptionalString(patch.description);
|
||||
}
|
||||
if ("displayName" in patch) {
|
||||
const displayName = normalizeOptionalString(patch.displayName);
|
||||
if (patch.displayName !== null && patch.displayName !== undefined && !displayName) {
|
||||
throw new Error("cron displayName must not be blank");
|
||||
}
|
||||
if (displayName && displayName.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
|
||||
throw new Error(
|
||||
`cron displayName must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
|
||||
);
|
||||
}
|
||||
const displayName = normalizeDeclarativeLabel(patch.displayName, "displayName", true);
|
||||
if (displayName) {
|
||||
job.displayName = displayName;
|
||||
} else {
|
||||
@@ -266,24 +342,7 @@ export function applyJobPatch(
|
||||
delete job.deleteAfterRun;
|
||||
}
|
||||
if (patch.schedule) {
|
||||
if (patch.schedule.kind === "cron") {
|
||||
const explicitStaggerMs = normalizeCronStaggerMs(patch.schedule.staggerMs);
|
||||
if (explicitStaggerMs !== undefined) {
|
||||
job.schedule = { ...patch.schedule, staggerMs: explicitStaggerMs };
|
||||
} else if (job.schedule.kind === "cron" && job.schedule.expr === patch.schedule.expr) {
|
||||
// Metadata-only resaves keep the existing stagger, but a replacement
|
||||
// expression owns a fresh default and must not inherit stale timing.
|
||||
job.schedule = { ...patch.schedule, staggerMs: job.schedule.staggerMs };
|
||||
} else {
|
||||
const defaultStaggerMs = resolveDefaultCronStaggerMs(patch.schedule.expr);
|
||||
job.schedule =
|
||||
defaultStaggerMs !== undefined
|
||||
? { ...patch.schedule, staggerMs: defaultStaggerMs }
|
||||
: patch.schedule;
|
||||
}
|
||||
} else {
|
||||
job.schedule = normalizeStreamScheduleBounds(patch.schedule);
|
||||
}
|
||||
job.schedule = normalizeJobSchedule(patch.schedule, { kind: "patch", previous: job.schedule });
|
||||
}
|
||||
if ("trigger" in patch) {
|
||||
if (patch.trigger === null || patch.trigger === undefined) {
|
||||
@@ -360,6 +419,7 @@ export function applyJobPatch(
|
||||
// alone owns the boundary that decides whether restart catch-up can run.
|
||||
delete statePatch.scheduleActivatedAtMs;
|
||||
delete statePatch.autoDisabled;
|
||||
assertCronJobStateTimestamps(statePatch);
|
||||
job.state = { ...job.state, ...statePatch };
|
||||
}
|
||||
if (patch.enabled === true) {
|
||||
@@ -389,34 +449,17 @@ export function applyJobPatch(
|
||||
job.state.streamLastStartedAtMs = undefined;
|
||||
job.state.streamLastExitAtMs = undefined;
|
||||
}
|
||||
assertSupportedJobSpec(job);
|
||||
assertPacingSupport(job);
|
||||
assertTriggerSupport(job, {
|
||||
cronConfig: opts?.cronConfig,
|
||||
requireEnabled: patch.trigger !== null && patch.trigger !== undefined,
|
||||
});
|
||||
assertScriptPayloadSupport(job, {
|
||||
cronConfig: opts?.cronConfig,
|
||||
requireEnabled: patch.payload?.kind === "script",
|
||||
// Enabled-only/rename patches must keep working on jobs stored with a
|
||||
// malformed script (pre-validation persistence); re-check syntax only
|
||||
// when this patch rewrites the payload, or disable becomes a dead end.
|
||||
validateSyntax: patch.payload !== undefined,
|
||||
});
|
||||
assertStreamScheduleSupport(job, {
|
||||
cronConfig: opts?.cronConfig,
|
||||
requireEnabled: patch.enabled === true || patch.schedule?.kind === "stream",
|
||||
});
|
||||
assertMainSessionAgentId(job, opts?.defaultAgentId);
|
||||
assertDeliverySupport(job);
|
||||
assertAnnounceDeliveryChannelSupport(job, opts?.configuredChannels, patch);
|
||||
assertFailureDestinationSupport(job);
|
||||
if (
|
||||
opts?.scheduleValidationNowMs !== undefined &&
|
||||
(patch.schedule !== undefined || patch.enabled === true)
|
||||
) {
|
||||
assertCronExpressionSatisfiable(job, opts.scheduleValidationNowMs, computeJobNextRunAtMs);
|
||||
}
|
||||
validateFullJob(
|
||||
job,
|
||||
{
|
||||
kind: "patch",
|
||||
patch,
|
||||
defaultAgentId: opts?.defaultAgentId,
|
||||
nowMs: opts?.scheduleValidationNowMs,
|
||||
cronConfig: opts?.cronConfig,
|
||||
},
|
||||
opts?.configuredChannels,
|
||||
);
|
||||
}
|
||||
|
||||
/** Converges the declared schedule, payload, delivery, and display label only. */
|
||||
@@ -438,44 +481,19 @@ export function applyDeclarativeJobSpec(
|
||||
const previousToolsAllowIsDefault = job.payload.toolsAllowIsDefault;
|
||||
// Name, target, routing, owner, and run policy remain outside declaration
|
||||
// convergence; changing those uses cron.update and cannot retarget an identity.
|
||||
const displayName = normalizeOptionalString(input.displayName);
|
||||
if (input.displayName !== undefined && !displayName) {
|
||||
throw new Error("cron displayName must not be blank");
|
||||
}
|
||||
if (displayName && displayName.length > CRON_DECLARATIVE_LABEL_MAX_LENGTH) {
|
||||
throw new Error(
|
||||
`cron displayName must be at most ${CRON_DECLARATIVE_LABEL_MAX_LENGTH} characters`,
|
||||
);
|
||||
}
|
||||
const displayName = normalizeDeclarativeLabel(input.displayName, "displayName");
|
||||
if (displayName) {
|
||||
job.displayName = displayName;
|
||||
} else {
|
||||
delete job.displayName;
|
||||
}
|
||||
|
||||
if (
|
||||
input.schedule.kind === "every" &&
|
||||
input.schedule.anchorMs === undefined &&
|
||||
job.schedule.kind === "every" &&
|
||||
job.schedule.everyMs === input.schedule.everyMs
|
||||
) {
|
||||
job.schedule = { ...input.schedule, anchorMs: job.schedule.anchorMs };
|
||||
} else if (input.schedule.kind === "every" && input.schedule.anchorMs === undefined) {
|
||||
job.schedule = { ...input.schedule, anchorMs: opts.nowMs };
|
||||
} else if (input.schedule.kind === "cron") {
|
||||
const explicitStaggerMs = normalizeCronStaggerMs(input.schedule.staggerMs);
|
||||
const defaultStaggerMs = resolveDefaultCronStaggerMs(input.schedule.expr);
|
||||
job.schedule = {
|
||||
...input.schedule,
|
||||
...(explicitStaggerMs !== undefined
|
||||
? { staggerMs: explicitStaggerMs }
|
||||
: defaultStaggerMs !== undefined
|
||||
? { staggerMs: defaultStaggerMs }
|
||||
: {}),
|
||||
};
|
||||
} else {
|
||||
job.schedule = normalizeStreamScheduleBounds(structuredClone(input.schedule));
|
||||
}
|
||||
job.schedule = normalizeJobSchedule(input.schedule, {
|
||||
kind: "declarative",
|
||||
previous: job.schedule,
|
||||
nowMs: opts.nowMs,
|
||||
fallbackAnchorMs: job.createdAtMs,
|
||||
});
|
||||
if (input.pacing !== undefined) {
|
||||
job.pacing = structuredClone(input.pacing);
|
||||
} else {
|
||||
@@ -523,26 +541,17 @@ export function applyDeclarativeJobSpec(
|
||||
if (opts.enabledExplicit) {
|
||||
job.enabled = input.enabled;
|
||||
}
|
||||
assertTriggerSupport(job, {
|
||||
cronConfig: opts.cronConfig,
|
||||
requireEnabled: input.trigger !== undefined,
|
||||
});
|
||||
assertScriptPayloadSupport(job, {
|
||||
cronConfig: opts.cronConfig,
|
||||
requireEnabled: input.payload.kind === "script",
|
||||
});
|
||||
assertStreamScheduleSupport(job, {
|
||||
cronConfig: opts.cronConfig,
|
||||
requireEnabled: true,
|
||||
});
|
||||
|
||||
assertSupportedJobSpec(job);
|
||||
assertPacingSupport(job);
|
||||
assertMainSessionAgentId(job, opts.defaultAgentId);
|
||||
assertDeliverySupport(job);
|
||||
assertAnnounceDeliveryChannelSupport(job, opts.configuredChannels);
|
||||
assertFailureDestinationSupport(job);
|
||||
assertCronExpressionSatisfiable(job, opts.nowMs, computeJobNextRunAtMs);
|
||||
validateFullJob(
|
||||
job,
|
||||
{
|
||||
kind: "declarative",
|
||||
input,
|
||||
defaultAgentId: opts.defaultAgentId,
|
||||
nowMs: opts.nowMs,
|
||||
cronConfig: opts.cronConfig,
|
||||
},
|
||||
opts.configuredChannels,
|
||||
);
|
||||
}
|
||||
|
||||
function mergeCronDelivery(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
import { nextWakeAtMs, recomputeNextRunsForMaintenance } from "./jobs.js";
|
||||
import { nextWakeAtMs, recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
import { emitCronRunFinished } from "./ops-run-preparation.js";
|
||||
import { cancelCronRunAdmissionWaiters } from "./run-admission.js";
|
||||
@@ -59,16 +59,22 @@ export async function start(state: CronServiceState) {
|
||||
...(finalized.triggerEval ? { triggerEval: finalized.triggerEval } : {}),
|
||||
deferredNotifications: postPersistNotifications,
|
||||
});
|
||||
// Skip only the old invocation; a distinct overdue replacement
|
||||
// must remain eligible for normal one-shot startup catch-up.
|
||||
if (repaired.replacementAtMs === undefined) {
|
||||
interruptedJobIds.add(job.id);
|
||||
if (repaired) {
|
||||
// Skip only the old invocation; a distinct overdue replacement
|
||||
// must remain eligible for normal one-shot startup catch-up.
|
||||
if (repaired.replacementAtMs === undefined) {
|
||||
interruptedJobIds.add(job.id);
|
||||
}
|
||||
if (repaired.shouldDelete) {
|
||||
completedJobIdsToDelete.add(job.id);
|
||||
}
|
||||
repairedAnyStartupRun = true;
|
||||
continue;
|
||||
}
|
||||
if (repaired.shouldDelete) {
|
||||
completedJobIdsToDelete.add(job.id);
|
||||
}
|
||||
repairedAnyStartupRun = true;
|
||||
continue;
|
||||
state.deps.log.warn(
|
||||
{ jobId: job.id },
|
||||
"cron: treating invalid finalized startup run as interrupted",
|
||||
);
|
||||
}
|
||||
const nowMs = state.deps.nowMs();
|
||||
const interrupted = markInterruptedStartupRun({
|
||||
|
||||
@@ -18,18 +18,16 @@ import { removeStaleCronJobFamilyRows } from "../store.js";
|
||||
import { createCronStreamSourceIdentity, cronStreamScheduleKey } from "../stream-schedule.js";
|
||||
import { normalizeCronTaskRunJobId } from "../task-run-history.js";
|
||||
import type { CronJob, CronJobCreate, CronJobPatch, CronStoredJob } from "../types.js";
|
||||
import { cronPatchTouchesDeliveryResolution } from "./jobs-validation.js";
|
||||
import {
|
||||
applyJobPatch,
|
||||
applyDeclarativeJobSpec,
|
||||
computeJobNextRunAtMs,
|
||||
createJob,
|
||||
findJobOrThrow,
|
||||
hasScheduledNextRunAtMs,
|
||||
isJobEnabled,
|
||||
nextWakeAtMs,
|
||||
recomputeNextRunsForMaintenance,
|
||||
} from "./jobs.js";
|
||||
} from "./jobs-scheduling.js";
|
||||
import { cronPatchTouchesDeliveryResolution } from "./jobs-validation.js";
|
||||
import { applyJobPatch, applyDeclarativeJobSpec, createJob } from "./jobs.js";
|
||||
import {
|
||||
getPendingCronSessionCleanup,
|
||||
locked,
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
import { resolveOpenClawStateSqlitePath } from "../../state/openclaw-state-db.paths.js";
|
||||
import { resolveCronListSnapshotRevision } from "../list-snapshot-revision.js";
|
||||
import { assertCronJobStateTimestamps } from "../persisted-shape.js";
|
||||
import { readCronJobScratchState, writeCronJobScratch } from "../scratch-store.js";
|
||||
import { createCronStreamSourceIdentity } from "../stream-schedule.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
import { findJobOrThrow, isJobEnabled, nextWakeAtMs, resolveJobLastRunStatus } from "./jobs.js";
|
||||
import {
|
||||
findJobOrThrow,
|
||||
isJobEnabled,
|
||||
nextWakeAtMs,
|
||||
resolveJobLastRunStatus,
|
||||
} from "./jobs-scheduling.js";
|
||||
import { sortCronJobs } from "./list-page-sort.js";
|
||||
import type {
|
||||
CronJobsEnabledFilter,
|
||||
@@ -113,6 +119,7 @@ export async function recordExternalFailure(
|
||||
const postPersistNotifications: DeferredCronNotifications = [];
|
||||
const now = state.deps.nowMs();
|
||||
const sourceIdentity = job.state.streamSourceIdentity;
|
||||
assertCronJobStateTimestamps(statePatch);
|
||||
Object.assign(job.state, statePatch);
|
||||
job.state.streamSourceIdentity = sourceIdentity;
|
||||
// Source restarts are counted separately, but terminal exhaustion should
|
||||
|
||||
@@ -5,17 +5,18 @@ import type { CronJob, CronPayload, CronRunErrorClassification } from "../types.
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
import {
|
||||
assertSupportedJobSpec,
|
||||
findJobOrThrow,
|
||||
hasActiveCronRun,
|
||||
isJobDue,
|
||||
isJobEnabled,
|
||||
recomputeNextRunsForMaintenance,
|
||||
} from "./jobs.js";
|
||||
} from "./jobs-scheduling.js";
|
||||
import { assertSupportedJobSpec } from "./jobs-validation.js";
|
||||
import { locked } from "./locked.js";
|
||||
import { markManualCronJobActive, ownsStreamSource } from "./ops-shared.js";
|
||||
import {
|
||||
activateQueuedCronRun,
|
||||
cleanupQueuedCronRunReservations,
|
||||
clearQueuedCronRunReservationMarker,
|
||||
isQueuedCronRunReservationCurrent,
|
||||
isQueuedCronRunReservationMarkerCurrent,
|
||||
@@ -559,20 +560,9 @@ export async function releasePreparedManualReservationAfterReloadWithRetry(
|
||||
state: CronServiceState,
|
||||
prepared: Extract<PreparedManualRun, { ran: true }>,
|
||||
): Promise<void> {
|
||||
const attempt = async () => {
|
||||
await locked(state, async () => {
|
||||
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
|
||||
await releasePreparedManualReservation(state, prepared);
|
||||
});
|
||||
};
|
||||
try {
|
||||
await attempt();
|
||||
} catch {
|
||||
try {
|
||||
await attempt();
|
||||
} catch (error) {
|
||||
releaseQueuedCronRun(state, prepared.jobId, prepared.reservationIdentity);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await cleanupQueuedCronRunReservations({
|
||||
state,
|
||||
reservations: [prepared],
|
||||
restoreLastError: false,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import { CommandLane } from "../../process/lanes.js";
|
||||
import { isCronActiveJobMarkerCurrent } from "../active-jobs.js";
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
import {
|
||||
activatePreparedManualRun,
|
||||
|
||||
@@ -3,7 +3,7 @@ import { parseAgentSessionKey } from "../../routing/session-key.js";
|
||||
import { clearCronJobActive, markCronJobActive, type CronActiveJobMarker } from "../active-jobs.js";
|
||||
import { cronStreamScheduleKey } from "../stream-schedule.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { normalizeOptionalAgentId } from "./normalize.js";
|
||||
import type { CronServiceState, DeferredCronNotifications } from "./state.js";
|
||||
import { ensureLoaded, persist } from "./store.js";
|
||||
|
||||
@@ -17,7 +17,7 @@ import { CommandLane } from "../../process/lanes.js";
|
||||
import * as cronStoreModule from "../store.js";
|
||||
import { loadCronStore, saveCronStore } from "../store.js";
|
||||
import { cronStreamScheduleKey } from "../stream-schedule.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { stop } from "./ops-lifecycle.js";
|
||||
import { update } from "./ops-mutations.js";
|
||||
import { list } from "./ops-read.js";
|
||||
|
||||
@@ -1,8 +1,19 @@
|
||||
// Shared execution admission for scheduled, manual, and on-exit cron runs.
|
||||
import { DEFAULT_CRON_MAX_CONCURRENT_RUNS } from "../../config/cron-limits.js";
|
||||
import { markCronJobActive } from "../active-jobs.js";
|
||||
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import type { CronServiceState } from "./state.js";
|
||||
import { persistOrRestore, snapshotStoreForRollback } from "./store.js";
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
|
||||
import { ensureLoaded, persistOrRestore, snapshotStoreForRollback } from "./store.js";
|
||||
import { tryCreateCronTaskRun } from "./task-runs.js";
|
||||
import {
|
||||
runsDetachedFromMainSession,
|
||||
type TimedCronRunOutcome,
|
||||
} from "./timer-execution-timeout.js";
|
||||
import { executeJobCoreWithTimeout } from "./timer-job-runner.js";
|
||||
import { isRunnableJob } from "./timer-runnable.js";
|
||||
|
||||
export function resolveRunConcurrency(): number {
|
||||
return DEFAULT_CRON_MAX_CONCURRENT_RUNS;
|
||||
@@ -95,7 +106,7 @@ export function isQueuedCronRunReservationCurrent(
|
||||
return state.queuedRunReservationsByJobId.get(jobId)?.identity === identity;
|
||||
}
|
||||
|
||||
export function restoreQueuedCronRunReservationLastError(
|
||||
function restoreQueuedCronRunReservationLastError(
|
||||
state: CronServiceState,
|
||||
jobId: string,
|
||||
identity: object,
|
||||
@@ -113,6 +124,7 @@ export function clearQueuedCronRunReservationMarker(
|
||||
jobId: string,
|
||||
identity: object,
|
||||
jobState: { queuedAtMs?: number; runningAtMs?: number; lastError?: string },
|
||||
opts?: { restoreLastError?: boolean },
|
||||
): boolean {
|
||||
const reservation = state.queuedRunReservationsByJobId.get(jobId);
|
||||
if (reservation?.identity !== identity) {
|
||||
@@ -123,7 +135,9 @@ export function clearQueuedCronRunReservationMarker(
|
||||
if (!queuedMatches && !runningMatches) {
|
||||
return false;
|
||||
}
|
||||
restoreQueuedCronRunReservationLastError(state, jobId, identity, jobState);
|
||||
if (opts?.restoreLastError !== false) {
|
||||
restoreQueuedCronRunReservationLastError(state, jobId, identity, jobState);
|
||||
}
|
||||
if (queuedMatches) {
|
||||
delete jobState.queuedAtMs;
|
||||
}
|
||||
@@ -133,6 +147,85 @@ export function clearQueuedCronRunReservationMarker(
|
||||
return true;
|
||||
}
|
||||
|
||||
type QueuedCronRunReservation = {
|
||||
jobId: string;
|
||||
reservationIdentity: object;
|
||||
};
|
||||
|
||||
export function clearOwnedQueuedCronRunMarkers(
|
||||
state: CronServiceState,
|
||||
reservations: readonly QueuedCronRunReservation[],
|
||||
opts?: { restoreLastError?: boolean },
|
||||
): QueuedCronRunReservation[] {
|
||||
const pendingReleases: QueuedCronRunReservation[] = [];
|
||||
for (const reservation of reservations) {
|
||||
const job = state.store?.jobs.find((entry) => entry.id === reservation.jobId);
|
||||
if (
|
||||
job &&
|
||||
clearQueuedCronRunReservationMarker(
|
||||
state,
|
||||
reservation.jobId,
|
||||
reservation.reservationIdentity,
|
||||
job.state,
|
||||
opts,
|
||||
)
|
||||
) {
|
||||
pendingReleases.push(reservation);
|
||||
} else {
|
||||
releaseQueuedCronRun(state, reservation.jobId, reservation.reservationIdentity);
|
||||
}
|
||||
}
|
||||
return pendingReleases;
|
||||
}
|
||||
|
||||
/** Durably clears reservations still owned by this process. Ownership stays
|
||||
* held through commit; after one retry it is dropped for restart repair. */
|
||||
export async function cleanupQueuedCronRunReservations(params: {
|
||||
state: CronServiceState;
|
||||
reservations: readonly QueuedCronRunReservation[];
|
||||
restoreLastError?: boolean;
|
||||
recompute?: "maintenance" | "startup-overflow";
|
||||
}): Promise<void> {
|
||||
const { state, reservations } = params;
|
||||
const attempt = async () => {
|
||||
await locked(state, async () => {
|
||||
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
const pendingReleases = clearOwnedQueuedCronRunMarkers(state, reservations, {
|
||||
restoreLastError: params.restoreLastError,
|
||||
});
|
||||
if (pendingReleases.length === 0) {
|
||||
return;
|
||||
}
|
||||
const postPersistNotifications: DeferredCronNotifications = [];
|
||||
if (params.recompute) {
|
||||
recomputeNextRunsForMaintenance(state, {
|
||||
...(params.recompute === "startup-overflow"
|
||||
? { repairFutureCronNextRunAtMs: false }
|
||||
: {}),
|
||||
deferredNotifications: postPersistNotifications,
|
||||
});
|
||||
}
|
||||
await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications });
|
||||
for (const reservation of pendingReleases) {
|
||||
releaseQueuedCronRun(state, reservation.jobId, reservation.reservationIdentity);
|
||||
}
|
||||
});
|
||||
};
|
||||
try {
|
||||
await attempt();
|
||||
} catch {
|
||||
try {
|
||||
await attempt();
|
||||
} catch (error) {
|
||||
for (const reservation of reservations) {
|
||||
releaseQueuedCronRun(state, reservation.jobId, reservation.reservationIdentity);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function isQueuedCronRunReservationMarkerCurrent(
|
||||
state: CronServiceState,
|
||||
jobId: string,
|
||||
@@ -189,8 +282,7 @@ export async function activateQueuedCronRun(params: {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply one service-level cap to every cron execution source. Queue waiters
|
||||
/** Apply one service-level cap to every cron execution source. Queue waiters
|
||||
* keep their job reservation, then recheck scheduler state before execution.
|
||||
*/
|
||||
export async function runWithCronAdmission<T>(
|
||||
@@ -207,3 +299,129 @@ export async function runWithCronAdmission<T>(
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function executeQueuedCronRun(params: {
|
||||
state: CronServiceState;
|
||||
jobId: string;
|
||||
reservedAtMs: number;
|
||||
reservationIdentity: object;
|
||||
runnableOptions?: Omit<Parameters<typeof isRunnableJob>[0], "state" | "job" | "nowMs">;
|
||||
isUnavailable?: () => boolean;
|
||||
onUnavailable?: () => void;
|
||||
onActivated?: () => void;
|
||||
onNotRunnable: (job: CronJob) => Promise<void>;
|
||||
onSetupError?: (job: CronJob, errorText: string) => void;
|
||||
/** Runs before admission release; true means terminal handling is complete. */
|
||||
onCompleted?: (outcome: TimedCronRunOutcome) => Promise<boolean>;
|
||||
}): Promise<
|
||||
| { kind: "stopped" }
|
||||
| { kind: "skipped" }
|
||||
| { kind: "completed"; outcome: TimedCronRunOutcome; handled: boolean }
|
||||
> {
|
||||
const { state } = params;
|
||||
let activated = false;
|
||||
const admission = await runWithCronAdmission(state, async () => {
|
||||
const started = await locked(state, async () => {
|
||||
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
|
||||
if (params.isUnavailable?.() || state.stopped || state.restartRecoveryPending) {
|
||||
params.onUnavailable?.();
|
||||
return undefined;
|
||||
}
|
||||
const job = state.store?.jobs.find((entry) => entry.id === params.jobId);
|
||||
if (
|
||||
!job ||
|
||||
!isQueuedCronRunReservationCurrent(state, params.jobId, params.reservationIdentity) ||
|
||||
job.state.queuedAtMs !== params.reservedAtMs
|
||||
) {
|
||||
releaseQueuedCronRun(state, params.jobId, params.reservationIdentity);
|
||||
return undefined;
|
||||
}
|
||||
const runnableJob = structuredClone(job);
|
||||
delete runnableJob.state.queuedAtMs;
|
||||
if (
|
||||
!isRunnableJob({
|
||||
state,
|
||||
job: runnableJob,
|
||||
nowMs: state.deps.nowMs(),
|
||||
...params.runnableOptions,
|
||||
})
|
||||
) {
|
||||
await params.onNotRunnable(job);
|
||||
return undefined;
|
||||
}
|
||||
const activation = await activateQueuedCronRun({
|
||||
state,
|
||||
job,
|
||||
reservationIdentity: params.reservationIdentity,
|
||||
onUnavailable: params.onUnavailable,
|
||||
});
|
||||
if (activation.kind !== "activated") {
|
||||
return undefined;
|
||||
}
|
||||
activated = true;
|
||||
params.onActivated?.();
|
||||
return { job, startedAt: activation.startedAt };
|
||||
});
|
||||
if (!started) {
|
||||
return undefined;
|
||||
}
|
||||
const executionJob = structuredClone(started.job);
|
||||
executionJob.state.runningAtMs = started.startedAt;
|
||||
executionJob.state.lastError = undefined;
|
||||
const taskRunId = tryCreateCronTaskRun({
|
||||
state,
|
||||
job: executionJob,
|
||||
startedAt: started.startedAt,
|
||||
});
|
||||
const activeJobMarker = markCronJobActive(executionJob.id, {
|
||||
preserveAcrossGenerationAdvance: !runsDetachedFromMainSession(executionJob),
|
||||
});
|
||||
emit(state, {
|
||||
jobId: executionJob.id,
|
||||
action: "started",
|
||||
job: executionJob,
|
||||
runAtMs: started.startedAt,
|
||||
});
|
||||
const base = {
|
||||
jobId: params.jobId,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
reservationIdentity: params.reservationIdentity,
|
||||
startedAt: started.startedAt,
|
||||
};
|
||||
let outcome: TimedCronRunOutcome;
|
||||
try {
|
||||
const result = await executeJobCoreWithTimeout(state, executionJob, {
|
||||
runId: taskRunId,
|
||||
activeJobMarker,
|
||||
});
|
||||
outcome = { ...base, ...result, endedAt: state.deps.nowMs() };
|
||||
} catch (error) {
|
||||
const errorText = normalizeCronRunErrorText(error);
|
||||
params.onSetupError?.(executionJob, errorText);
|
||||
outcome = {
|
||||
...base,
|
||||
status: "error",
|
||||
error: errorText,
|
||||
diagnostics: createCronRunDiagnosticsFromError("cron-setup", errorText, {
|
||||
nowMs: state.deps.nowMs,
|
||||
}),
|
||||
endedAt: state.deps.nowMs(),
|
||||
};
|
||||
}
|
||||
return { outcome, handled: (await params.onCompleted?.(outcome)) === true };
|
||||
}).catch((error: unknown) => {
|
||||
if (activated) {
|
||||
releaseQueuedCronRun(state, params.jobId, params.reservationIdentity);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
if (admission.kind === "stopped") {
|
||||
return { kind: "stopped" };
|
||||
}
|
||||
if (!admission.value) {
|
||||
return { kind: "skipped" };
|
||||
}
|
||||
return { kind: "completed", ...admission.value };
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { markInterruptedStartupRun } from "./startup-run-repair.js";
|
||||
import { markInterruptedStartupRun, restoreFinalizedStartupRun } from "./startup-run-repair.js";
|
||||
import { createCronServiceState } from "./state.js";
|
||||
|
||||
describe("startup run repair auto-disable", () => {
|
||||
@@ -68,4 +69,117 @@ describe("startup run repair auto-disable", () => {
|
||||
expect(enqueueSystemEvent).toHaveBeenCalledOnce();
|
||||
expect(requestHeartbeat).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("disables a job instead of restoring an invalid finalized next run", () => {
|
||||
const runningAtMs = Date.parse("2026-08-01T16:00:00.000Z");
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/startup-run-repair-invalid-next-run.json",
|
||||
cronEnabled: true,
|
||||
log: {
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
nowMs: () => runningAtMs + 1_000,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(),
|
||||
});
|
||||
const job: CronJob = {
|
||||
id: "invalid-finalized-next-run",
|
||||
name: "invalid finalized next run",
|
||||
enabled: true,
|
||||
createdAtMs: runningAtMs - 60_000,
|
||||
updatedAtMs: runningAtMs,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: runningAtMs - 60_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "do not replay" },
|
||||
state: { nextRunAtMs: runningAtMs, runningAtMs },
|
||||
};
|
||||
const deferredNotifications: Array<() => void> = [];
|
||||
|
||||
restoreFinalizedStartupRun({
|
||||
state,
|
||||
job,
|
||||
runningAtMs,
|
||||
deferredNotifications,
|
||||
entry: {
|
||||
ts: runningAtMs + 1_000,
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
runAtMs: runningAtMs,
|
||||
durationMs: 1_000,
|
||||
nextRunAtMs: MAX_DATE_TIMESTAMP_MS + 1,
|
||||
},
|
||||
});
|
||||
|
||||
expect(job.enabled).toBe(false);
|
||||
expect(job.state.nextRunAtMs).toBeUndefined();
|
||||
expect(job.state.autoDisabled).toEqual({
|
||||
reason: "schedule-errors",
|
||||
atMs: runningAtMs + 1_000,
|
||||
consecutiveErrors: 1,
|
||||
});
|
||||
expect(state.deps.enqueueSystemEvent).not.toHaveBeenCalled();
|
||||
expect(state.deps.requestHeartbeat).not.toHaveBeenCalled();
|
||||
expect(deferredNotifications).toHaveLength(1);
|
||||
|
||||
deferredNotifications[0]?.();
|
||||
expect(state.deps.enqueueSystemEvent).toHaveBeenCalledOnce();
|
||||
expect(state.deps.requestHeartbeat).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each(["runAtMs", "ts"] as const)(
|
||||
"ignores finalized startup history with an invalid %s",
|
||||
(field) => {
|
||||
const runningAtMs = Date.parse("2026-08-01T16:00:00.000Z");
|
||||
const warn = vi.fn();
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/startup-run-repair-invalid-history.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() },
|
||||
nowMs: () => runningAtMs + 1_000,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(),
|
||||
});
|
||||
const job: CronJob = {
|
||||
id: "invalid-finalized-history",
|
||||
name: "invalid finalized history",
|
||||
enabled: true,
|
||||
createdAtMs: runningAtMs - 60_000,
|
||||
updatedAtMs: runningAtMs,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "do not replay" },
|
||||
state: { nextRunAtMs: runningAtMs, runningAtMs },
|
||||
};
|
||||
const before = structuredClone(job);
|
||||
|
||||
const result = restoreFinalizedStartupRun({
|
||||
state,
|
||||
job,
|
||||
runningAtMs,
|
||||
entry: {
|
||||
ts: field === "ts" ? MAX_DATE_TIMESTAMP_MS + 1 : runningAtMs + 1_000,
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
runAtMs: field === "runAtMs" ? MAX_DATE_TIMESTAMP_MS + 1 : runningAtMs,
|
||||
durationMs: 1_000,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(job).toEqual(before);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
{ jobId: job.id },
|
||||
"cron: ignoring finalized startup run with an invalid timestamp envelope",
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
/** Repairs interrupted and finalized cron runs while the service starts. */
|
||||
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
|
||||
import { resolveCronDeliveryPlan, resolveFailureDestination } from "../delivery-plan.js";
|
||||
import { parseAbsoluteTimeMs } from "../parse.js";
|
||||
import type { CronRunLogEntry } from "../run-log-types.js";
|
||||
import type { CronJob, CronRunStatus } from "../types.js";
|
||||
import { maybeAutoDisableCronJobAfterRunFailure } from "./auto-disable.js";
|
||||
import type { CronServiceState, DeferredCronNotifications } from "./state.js";
|
||||
import { resolveNextRunAtMsOrDisable } from "./timer-trigger.js";
|
||||
import {
|
||||
applyJobResult,
|
||||
applyScriptRunResult,
|
||||
@@ -127,9 +129,17 @@ export function restoreFinalizedStartupRun(params: {
|
||||
scriptResult?: { scriptStateChanged: true; scriptState?: unknown };
|
||||
triggerEval?: CronTriggerEvalOutcome;
|
||||
deferredNotifications?: DeferredCronNotifications;
|
||||
}): { shouldDelete: boolean; replacementAtMs?: number } {
|
||||
}): { shouldDelete: boolean; replacementAtMs?: number } | undefined {
|
||||
const { state, job, runningAtMs, entry } = params;
|
||||
const startedAt = entry.runAtMs ?? runningAtMs;
|
||||
const startedAt = asDateTimestampMs(entry.runAtMs ?? runningAtMs);
|
||||
const endedAt = asDateTimestampMs(entry.ts);
|
||||
if (startedAt === undefined || startedAt < 0 || endedAt === undefined || endedAt < 0) {
|
||||
state.deps.log.warn(
|
||||
{ jobId: job.id },
|
||||
"cron: ignoring finalized startup run with an invalid timestamp envelope",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
const replacementAtMs = resolveOneShotReplacementAtMs(job, startedAt);
|
||||
const scheduleOwnership = replacementAtMs === undefined ? "current" : "stale";
|
||||
const shouldDelete = applyJobResult(
|
||||
@@ -138,17 +148,17 @@ export function restoreFinalizedStartupRun(params: {
|
||||
{
|
||||
...entry,
|
||||
startedAt,
|
||||
endedAt: entry.ts,
|
||||
endedAt,
|
||||
},
|
||||
{
|
||||
replayFailureAlertAtMs: entry.ts,
|
||||
replayFailureAlertAtMs: endedAt,
|
||||
scheduleOwnership,
|
||||
deferredNotifications: params.deferredNotifications,
|
||||
},
|
||||
);
|
||||
|
||||
// The finalized row captured post-run state before the stale cron store write.
|
||||
job.state.lastDurationMs = entry.durationMs ?? Math.max(0, entry.ts - startedAt);
|
||||
job.state.lastDurationMs = entry.durationMs ?? Math.max(0, endedAt - startedAt);
|
||||
job.state.lastErrorReason = entry.errorReason;
|
||||
job.state.lastDelivered = entry.delivered;
|
||||
job.state.lastDeliveryStatus = entry.deliveryStatus;
|
||||
@@ -156,9 +166,16 @@ export function restoreFinalizedStartupRun(params: {
|
||||
job.state.lastFailureNotificationDelivered = entry.failureNotificationDelivery?.delivered;
|
||||
job.state.lastFailureNotificationDeliveryStatus = entry.failureNotificationDelivery?.status;
|
||||
job.state.lastFailureNotificationDeliveryError = entry.failureNotificationDelivery?.error;
|
||||
job.state.nextRunAtMs = job.state.autoDisabled
|
||||
? undefined
|
||||
: (replacementAtMs ?? entry.nextRunAtMs);
|
||||
const finalizedNextRunAtMs = replacementAtMs ?? entry.nextRunAtMs;
|
||||
job.state.nextRunAtMs =
|
||||
job.state.autoDisabled || finalizedNextRunAtMs === undefined
|
||||
? undefined
|
||||
: resolveNextRunAtMsOrDisable({
|
||||
state,
|
||||
job,
|
||||
candidate: finalizedNextRunAtMs,
|
||||
deferredNotifications: params.deferredNotifications,
|
||||
});
|
||||
// The finalized ledger row owns the schedule decision made before the stale
|
||||
// store write. No next run means that one-shot was permanently disabled.
|
||||
if (
|
||||
@@ -173,7 +190,7 @@ export function restoreFinalizedStartupRun(params: {
|
||||
job,
|
||||
{
|
||||
status: entry.status,
|
||||
endedAt: entry.ts,
|
||||
endedAt,
|
||||
triggerEval: params.triggerEval,
|
||||
},
|
||||
{ scheduleOwnership },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Missing session target tests cover loading legacy cron state without targets.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assertSupportedJobSpec } from "./jobs.js";
|
||||
import { assertSupportedJobSpec } from "./jobs-validation.js";
|
||||
|
||||
describe("cron service store load: missing sessionTarget", () => {
|
||||
it("assertSupportedJobSpec throws a clear error when sessionTarget is missing", () => {
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
// Cron service store tests cover persisted service state loading and writes.
|
||||
import fs from "node:fs/promises";
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { openOpenClawStateDatabase } from "../../state/openclaw-state-db.js";
|
||||
import { setupCronServiceSuite } from "../service.test-harness.js";
|
||||
import * as cronStoreModule from "../store.js";
|
||||
import { loadCronStore, saveCronStore } from "../store.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { findJobOrThrow } from "./jobs.js";
|
||||
import { findJobOrThrow } from "./jobs-scheduling.js";
|
||||
import { createCronServiceState } from "./state.js";
|
||||
import { ensureLoaded, persist, persistOrRestore, snapshotStoreForRollback } from "./store.js";
|
||||
|
||||
@@ -153,6 +154,98 @@ describe("cron service store seam coverage", () => {
|
||||
await expectPathMissing(storePath.replace(/\.json$/, "-quarantine.json"));
|
||||
});
|
||||
|
||||
it("quarantines persisted every schedules that cannot produce valid Date timestamps", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const invalidInterval = createReloadCronJob({
|
||||
id: "invalid-date-interval",
|
||||
schedule: { kind: "every", everyMs: 1_000 },
|
||||
});
|
||||
const invalidAnchor = createReloadCronJob({
|
||||
id: "invalid-date-anchor",
|
||||
schedule: { kind: "every", everyMs: 1_000, anchorMs: 0 },
|
||||
});
|
||||
const unsatisfiableInterval = createReloadCronJob({
|
||||
id: "unsatisfiable-date-interval",
|
||||
schedule: { kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS },
|
||||
});
|
||||
const disabledUnsatisfiableInterval = createReloadCronJob({
|
||||
id: "disabled-unsatisfiable-date-interval",
|
||||
enabled: false,
|
||||
schedule: { kind: "every", everyMs: MAX_DATE_TIMESTAMP_MS },
|
||||
});
|
||||
const invalidStagger = createReloadCronJob({ id: "invalid-date-stagger" });
|
||||
const repairableState = createReloadCronJob({
|
||||
id: "repairable-runtime-state",
|
||||
state: { lastRunAtMs: MAX_DATE_TIMESTAMP_MS },
|
||||
});
|
||||
const surviving = createReloadCronJob({ id: "valid-schedule" });
|
||||
await saveCronStore(storePath, {
|
||||
version: 1,
|
||||
jobs: [
|
||||
invalidInterval,
|
||||
invalidAnchor,
|
||||
unsatisfiableInterval,
|
||||
disabledUnsatisfiableInterval,
|
||||
invalidStagger,
|
||||
repairableState,
|
||||
surviving,
|
||||
],
|
||||
});
|
||||
const db = openOpenClawStateDatabase().db;
|
||||
db.prepare("UPDATE cron_jobs SET every_ms = ? WHERE job_id = ?").run(
|
||||
MAX_DATE_TIMESTAMP_MS + 1,
|
||||
invalidInterval.id,
|
||||
);
|
||||
db.prepare("UPDATE cron_jobs SET anchor_ms = ? WHERE job_id = ?").run(
|
||||
MAX_DATE_TIMESTAMP_MS + 1,
|
||||
invalidAnchor.id,
|
||||
);
|
||||
db.prepare("UPDATE cron_jobs SET stagger_ms = ? WHERE job_id = ?").run(
|
||||
MAX_DATE_TIMESTAMP_MS + 1,
|
||||
invalidStagger.id,
|
||||
);
|
||||
const state = createStoreTestState(storePath);
|
||||
|
||||
await ensureLoaded(state, { skipRecompute: true });
|
||||
|
||||
expect(state.store?.jobs.map((job) => job.id)).toEqual([
|
||||
disabledUnsatisfiableInterval.id,
|
||||
repairableState.id,
|
||||
surviving.id,
|
||||
]);
|
||||
expect(cronStoreModule.loadCronQuarantinedJobs(storePath)).toEqual([
|
||||
expect.objectContaining({ job: expect.objectContaining({ id: invalidInterval.id }) }),
|
||||
expect.objectContaining({ job: expect.objectContaining({ id: invalidAnchor.id }) }),
|
||||
expect.objectContaining({
|
||||
reason: "unsatisfiable-schedule",
|
||||
job: expect.objectContaining({ id: unsatisfiableInterval.id }),
|
||||
}),
|
||||
expect.objectContaining({ job: expect.objectContaining({ id: invalidStagger.id }) }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("quarantines persisted runtime timestamps outside the Date domain", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const invalidState = createReloadCronJob({ id: "invalid-runtime-state" });
|
||||
const surviving = createReloadCronJob({ id: "valid-runtime-state" });
|
||||
await saveCronStore(storePath, { version: 1, jobs: [invalidState, surviving] });
|
||||
openOpenClawStateDatabase()
|
||||
.db.prepare("UPDATE cron_jobs SET last_run_at_ms = ? WHERE job_id = ?")
|
||||
.run(MAX_DATE_TIMESTAMP_MS + 1, invalidState.id);
|
||||
const state = createStoreTestState(storePath);
|
||||
|
||||
await ensureLoaded(state, { skipRecompute: true });
|
||||
|
||||
expect(state.store?.jobs.map((job) => job.id)).toEqual([surviving.id]);
|
||||
expect(cronStoreModule.loadCronQuarantinedJobs(storePath)).toEqual([
|
||||
expect.objectContaining({
|
||||
reason: "invalid-state",
|
||||
job: expect.objectContaining({ id: invalidState.id }),
|
||||
state: expect.objectContaining({ lastRunAtMs: MAX_DATE_TIMESTAMP_MS + 1 }),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("publishes durable wake changes only after save and exactly once after retry", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const initialNextRunAtMs = STORE_TEST_NOW + 60_000;
|
||||
|
||||
@@ -12,7 +12,8 @@ import {
|
||||
type QuarantinedCronConfigJob,
|
||||
} from "../store.js";
|
||||
import type { CronJob, CronStoreFile } from "../types.js";
|
||||
import { recomputeNextRuns } from "./jobs.js";
|
||||
import { computeJobNextRunAtMs, recomputeNextRuns } from "./jobs-scheduling.js";
|
||||
import { assertTimeScheduleSatisfiable } from "./jobs-validation.js";
|
||||
import { emit, type CronServiceState, type DeferredCronNotifications } from "./state.js";
|
||||
|
||||
const loadedCronStoreRevisions = new WeakMap<CronServiceState, number>();
|
||||
@@ -142,6 +143,7 @@ export async function ensureLoaded(
|
||||
previousJobsById.set(job.id, job);
|
||||
}
|
||||
const loaded = await loadCronJobsStoreWithConfigJobs(state.deps.storePath);
|
||||
const loadNowMs = state.deps.nowMs();
|
||||
// Persisted cron rows are validated lazily, so treat them as raw records at the
|
||||
// store boundary and only trust the CronJob shape after validation below.
|
||||
const loadedJobs = (loaded.store.jobs ?? []) as unknown as Record<string, unknown>[];
|
||||
@@ -155,6 +157,7 @@ export async function ensureLoaded(
|
||||
// Accept old `jobId` rows at the raw boundary only; the in-memory store
|
||||
// uses canonical `id` before validation and scheduling.
|
||||
normalizeCronJobIdentityFields(raw);
|
||||
const rawInvalidReason = getInvalidPersistedCronJobReason(raw);
|
||||
let normalized: Record<string, unknown> | null;
|
||||
try {
|
||||
normalized = normalizeCronJobInput(raw);
|
||||
@@ -169,7 +172,19 @@ export async function ensureLoaded(
|
||||
);
|
||||
}
|
||||
const hydratedRaw = normalized ?? raw;
|
||||
const invalidReason = getInvalidPersistedCronJobReason(hydratedRaw);
|
||||
let invalidReason = rawInvalidReason ?? getInvalidPersistedCronJobReason(hydratedRaw);
|
||||
const hydratedSchedule = (hydratedRaw.schedule ?? {}) as Record<string, unknown>;
|
||||
if (!invalidReason && hydratedRaw.enabled !== false && hydratedSchedule.kind === "every") {
|
||||
try {
|
||||
assertTimeScheduleSatisfiable(
|
||||
{ ...(hydratedRaw as unknown as CronJob), state: {} },
|
||||
loadNowMs,
|
||||
computeJobNextRunAtMs,
|
||||
);
|
||||
} catch {
|
||||
invalidReason = "unsatisfiable-schedule";
|
||||
}
|
||||
}
|
||||
if (invalidReason) {
|
||||
const quarantineEntry: QuarantinedCronConfigJob = {
|
||||
sourceIndex,
|
||||
@@ -206,7 +221,7 @@ export async function ensureLoaded(
|
||||
jobs,
|
||||
};
|
||||
state.durableNextRunAtMsByJobId = durableNextRunAtMsByJobId;
|
||||
state.storeLoadedAtMs = state.deps.nowMs();
|
||||
state.storeLoadedAtMs = loadNowMs;
|
||||
loadedCronStoreRevisions.set(state, getCronJobsStoreRevision(state.deps.storePath));
|
||||
|
||||
if (quarantinedConfigJobs.length > 0) {
|
||||
|
||||
@@ -1,45 +1,33 @@
|
||||
import { markCronJobActive } from "../active-jobs.js";
|
||||
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import {
|
||||
DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
|
||||
isJobEnabled,
|
||||
recomputeNextRunsForMaintenance,
|
||||
resolveJobErrorBackoffUntilMs,
|
||||
} from "./jobs.js";
|
||||
} from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
import {
|
||||
activateQueuedCronRun,
|
||||
isQueuedCronRunReservationCurrent,
|
||||
clearOwnedQueuedCronRunMarkers,
|
||||
cleanupQueuedCronRunReservations,
|
||||
executeQueuedCronRun,
|
||||
releaseQueuedCronRun,
|
||||
reserveQueuedCronRun,
|
||||
runWithCronAdmission,
|
||||
} from "./run-admission.js";
|
||||
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
|
||||
import type { CronServiceState, DeferredCronNotifications } from "./state.js";
|
||||
import { ensureLoaded, persist, persistOrRestore, snapshotStoreForRollback } from "./store.js";
|
||||
import { tryCreateCronTaskRun } from "./task-runs.js";
|
||||
import {
|
||||
DEFAULT_MAX_MISSED_JOBS_PER_RESTART,
|
||||
DEFAULT_MISSED_JOB_STAGGER_MS,
|
||||
DEFAULT_STARTUP_DEFERRED_MISSED_AGENT_JOB_DELAY_MS,
|
||||
runsDetachedFromMainSession,
|
||||
type StartupCatchupCandidate,
|
||||
type StartupCatchupExecution,
|
||||
type StartupCatchupPlan,
|
||||
type StartupDeferredJob,
|
||||
type TimedCronRunOutcome,
|
||||
} from "./timer-execution-timeout.js";
|
||||
import { executeJobCoreWithTimeout } from "./timer-job-runner.js";
|
||||
import {
|
||||
clearUnstartedStartupCatchupReservationMarkers,
|
||||
createCompletedCronRunOutcomeDrain,
|
||||
} from "./timer-outcome-finalization.js";
|
||||
import {
|
||||
collectRunnableJobs,
|
||||
hasMissedCronSlotSinceLastRun,
|
||||
isRunnableJob,
|
||||
} from "./timer-runnable.js";
|
||||
import { createCompletedCronRunOutcomeDrain } from "./timer-outcome-finalization.js";
|
||||
import { collectRunnableJobs, hasMissedCronSlotSinceLastRun } from "./timer-runnable.js";
|
||||
import { maybeNotifyIsolatedAgentSetupTimeout } from "./timer-scheduler.js";
|
||||
import { resolveNextRunAtMsOrDisable } from "./timer-trigger.js";
|
||||
|
||||
function deferPendingBackoffMissedCronSlots(
|
||||
state: CronServiceState,
|
||||
@@ -79,8 +67,8 @@ async function persistStartupCatchupReservations(
|
||||
state: CronServiceState,
|
||||
rollbackSnapshot: ReturnType<typeof snapshotStoreForRollback>,
|
||||
pendingReleases: readonly Pick<StartupCatchupCandidate, "jobId" | "reservationIdentity">[],
|
||||
postPersistNotifications: DeferredCronNotifications = [],
|
||||
): Promise<void> {
|
||||
const postPersistNotifications: DeferredCronNotifications = [];
|
||||
recomputeNextRunsForMaintenance(state, {
|
||||
repairFutureCronNextRunAtMs: false,
|
||||
deferredNotifications: postPersistNotifications,
|
||||
@@ -97,31 +85,12 @@ async function releaseStartupCatchupReservationsAfterFailure(
|
||||
plan: StartupCatchupPlan,
|
||||
outcomes: readonly TimedCronRunOutcome[],
|
||||
): Promise<void> {
|
||||
const attempt = async () => {
|
||||
await locked(state, async () => {
|
||||
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
const pendingReleases = clearUnstartedStartupCatchupReservationMarkers(state, plan, outcomes);
|
||||
if (pendingReleases.length === 0) {
|
||||
return;
|
||||
}
|
||||
await persistStartupCatchupReservations(state, rollbackSnapshot, pendingReleases);
|
||||
});
|
||||
};
|
||||
try {
|
||||
await attempt();
|
||||
} catch {
|
||||
try {
|
||||
await attempt();
|
||||
} catch (error) {
|
||||
// The failed execution has no remaining cleanup owner. Release process
|
||||
// claims so durable stuck-marker recovery can eventually repair them.
|
||||
for (const candidate of plan.candidates) {
|
||||
releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const startedJobIds = new Set(outcomes.map((outcome) => outcome.jobId));
|
||||
await cleanupQueuedCronRunReservations({
|
||||
state,
|
||||
reservations: plan.candidates.filter((candidate) => !startedJobIds.has(candidate.jobId)),
|
||||
recompute: "startup-overflow",
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs or defers missed startup jobs using restart catch-up limits. */
|
||||
@@ -281,69 +250,29 @@ async function executeStartupCatchupPlan(
|
||||
if (state.stopped) {
|
||||
break;
|
||||
}
|
||||
const admission = await runWithCronAdmission(state, async () => {
|
||||
const startedCandidate = await locked(state, async () => {
|
||||
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
|
||||
const job = state.store?.jobs.find((entry) => entry.id === candidate.jobId);
|
||||
if (state.stopped || state.restartRecoveryPending) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
!job ||
|
||||
!isQueuedCronRunReservationCurrent(
|
||||
state,
|
||||
candidate.jobId,
|
||||
candidate.reservationIdentity,
|
||||
) ||
|
||||
job.state.queuedAtMs !== candidate.reservedAtMs
|
||||
) {
|
||||
releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity);
|
||||
return undefined;
|
||||
}
|
||||
const dueProbe = structuredClone(job);
|
||||
delete dueProbe.state.queuedAtMs;
|
||||
if (
|
||||
!isRunnableJob({
|
||||
state,
|
||||
job: dueProbe,
|
||||
nowMs: state.deps.nowMs(),
|
||||
skipAtIfAlreadyRan: true,
|
||||
allowCronMissedRunByLastRun: true,
|
||||
})
|
||||
) {
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
delete job.state.queuedAtMs;
|
||||
await persistStartupCatchupReservations(state, rollbackSnapshot, [candidate]);
|
||||
return undefined;
|
||||
}
|
||||
const activation = await activateQueuedCronRun({
|
||||
state,
|
||||
job,
|
||||
reservationIdentity: candidate.reservationIdentity,
|
||||
});
|
||||
if (activation.kind === "unavailable") {
|
||||
return undefined;
|
||||
}
|
||||
return { ...candidate, job, startedAt: activation.startedAt };
|
||||
});
|
||||
if (!startedCandidate) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return await runStartupCatchupCandidate(state, startedCandidate);
|
||||
} catch (error) {
|
||||
releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity);
|
||||
throw error;
|
||||
}
|
||||
const execution = await executeQueuedCronRun({
|
||||
state,
|
||||
jobId: candidate.jobId,
|
||||
reservedAtMs: candidate.reservedAtMs,
|
||||
reservationIdentity: candidate.reservationIdentity,
|
||||
runnableOptions: {
|
||||
skipAtIfAlreadyRan: true,
|
||||
allowCronMissedRunByLastRun: true,
|
||||
},
|
||||
onNotRunnable: async (job) => {
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
delete job.state.queuedAtMs;
|
||||
await persistStartupCatchupReservations(state, rollbackSnapshot, [candidate]);
|
||||
},
|
||||
});
|
||||
if (admission.kind === "stopped") {
|
||||
if (execution.kind === "stopped") {
|
||||
break;
|
||||
}
|
||||
if (admission.value) {
|
||||
if (execution.kind === "completed") {
|
||||
// Catch-up execution stays sequential, while completed outcomes
|
||||
// persist in coalesced batches before slower siblings have to drain.
|
||||
outcomes.push(admission.value);
|
||||
completedOutcomeDrain.enqueue(admission.value);
|
||||
outcomes.push(execution.outcome);
|
||||
completedOutcomeDrain.enqueue(execution.outcome);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -352,62 +281,6 @@ async function executeStartupCatchupPlan(
|
||||
return { ok: true, outcomes };
|
||||
}
|
||||
|
||||
async function runStartupCatchupCandidate(
|
||||
state: CronServiceState,
|
||||
candidate: StartupCatchupCandidate & { startedAt: number },
|
||||
): Promise<TimedCronRunOutcome> {
|
||||
const { startedAt } = candidate;
|
||||
const executionJob = structuredClone(candidate.job);
|
||||
executionJob.state.runningAtMs = startedAt;
|
||||
const taskRunId = tryCreateCronTaskRun({
|
||||
state,
|
||||
job: executionJob,
|
||||
startedAt,
|
||||
});
|
||||
const activeJobMarker = markCronJobActive(executionJob.id, {
|
||||
preserveAcrossGenerationAdvance: !runsDetachedFromMainSession(executionJob),
|
||||
});
|
||||
emit(state, {
|
||||
jobId: executionJob.id,
|
||||
action: "started",
|
||||
job: executionJob,
|
||||
runAtMs: startedAt,
|
||||
});
|
||||
try {
|
||||
const result = await executeJobCoreWithTimeout(state, executionJob, {
|
||||
runId: taskRunId,
|
||||
activeJobMarker,
|
||||
});
|
||||
return {
|
||||
jobId: candidate.jobId,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
reservationIdentity: candidate.reservationIdentity,
|
||||
// Keep the complete core outcome: startup catch-up shares the same result
|
||||
// application path as timer runs, including delivery and script state.
|
||||
...result,
|
||||
startedAt,
|
||||
endedAt: state.deps.nowMs(),
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
jobId: candidate.jobId,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
reservationIdentity: candidate.reservationIdentity,
|
||||
status: "error",
|
||||
error: normalizeCronRunErrorText(err),
|
||||
diagnostics: createCronRunDiagnosticsFromError("cron-setup", normalizeCronRunErrorText(err), {
|
||||
nowMs: state.deps.nowMs,
|
||||
}),
|
||||
startedAt,
|
||||
endedAt: state.deps.nowMs(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function applyStartupCatchupOutcomes(
|
||||
state: CronServiceState,
|
||||
plan: StartupCatchupPlan,
|
||||
@@ -422,7 +295,12 @@ async function applyStartupCatchupOutcomes(
|
||||
return;
|
||||
}
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
const pendingReleases = clearUnstartedStartupCatchupReservationMarkers(state, plan, outcomes);
|
||||
const startedJobIds = new Set(outcomes.map((outcome) => outcome.jobId));
|
||||
const pendingReleases = clearOwnedQueuedCronRunMarkers(
|
||||
state,
|
||||
plan.candidates.filter((candidate) => !startedJobIds.has(candidate.jobId)),
|
||||
);
|
||||
const postPersistNotifications: DeferredCronNotifications = [];
|
||||
if (state.stopped || (outcomes.length === 0 && plan.deferredJobs.length === 0)) {
|
||||
if (pendingReleases.length > 0) {
|
||||
await persistStartupCatchupReservations(state, rollbackSnapshot, pendingReleases);
|
||||
@@ -440,13 +318,23 @@ async function applyStartupCatchupOutcomes(
|
||||
continue;
|
||||
}
|
||||
if (typeof deferred.delayMs === "number") {
|
||||
const runAtMs = baseNow + deferred.delayMs + offset - staggerMs;
|
||||
const runAtMs = resolveNextRunAtMsOrDisable({
|
||||
state,
|
||||
job,
|
||||
candidate: baseNow + deferred.delayMs + offset - staggerMs,
|
||||
deferredNotifications: postPersistNotifications,
|
||||
});
|
||||
job.state.nextRunAtMs = runAtMs;
|
||||
job.state.startupCatchupAtMs = runAtMs;
|
||||
offset += staggerMs;
|
||||
continue;
|
||||
}
|
||||
const runAtMs = baseNow + offset;
|
||||
const runAtMs = resolveNextRunAtMsOrDisable({
|
||||
state,
|
||||
job,
|
||||
candidate: baseNow + offset,
|
||||
deferredNotifications: postPersistNotifications,
|
||||
});
|
||||
job.state.nextRunAtMs = runAtMs;
|
||||
job.state.startupCatchupAtMs = runAtMs;
|
||||
offset += staggerMs;
|
||||
@@ -455,7 +343,12 @@ async function applyStartupCatchupOutcomes(
|
||||
|
||||
// Startup overflow owns these staggered wake times; repairing future
|
||||
// schedules here would silently move a deferred run to its natural slot.
|
||||
await persistStartupCatchupReservations(state, rollbackSnapshot, pendingReleases);
|
||||
await persistStartupCatchupReservations(
|
||||
state,
|
||||
rollbackSnapshot,
|
||||
pendingReleases,
|
||||
postPersistNotifications,
|
||||
);
|
||||
});
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import type {
|
||||
CronRunTelemetry,
|
||||
} from "../types.js";
|
||||
import { abortErrorMessage, timeoutErrorMessage } from "./execution-errors.js";
|
||||
import { resolveJobPayloadTextForMain } from "./jobs.js";
|
||||
import { resolveJobPayloadTextForMain } from "./jobs-scheduling.js";
|
||||
import type { CronServiceState } from "./state.js";
|
||||
import { resolveMainSessionCronRunSessionKey } from "./task-runs.js";
|
||||
import {
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
import { clearCronJobActive, isCronActiveJobMarkerCurrent } from "../active-jobs.js";
|
||||
import type { CronActiveJobMarker } from "../active-jobs.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
import { clearQueuedCronRunReservationMarker, releaseQueuedCronRun } from "./run-admission.js";
|
||||
import { releaseQueuedCronRun } from "./run-admission.js";
|
||||
import { emit, type CronServiceState, type DeferredCronNotifications } from "./state.js";
|
||||
import {
|
||||
ensureLoaded,
|
||||
@@ -28,10 +28,6 @@ type CronTaskRunFinalizationOutcome = {
|
||||
activeJobMarker?: CronActiveJobMarker;
|
||||
};
|
||||
|
||||
type StartupCatchupReservationPlan = {
|
||||
candidates: readonly { jobId: string; reservedAtMs: number; reservationIdentity: object }[];
|
||||
};
|
||||
|
||||
type CompletedCronRunOutcomeFinalizationOptions = {
|
||||
clearOnFailure?: boolean;
|
||||
discardWhenStopped?: boolean;
|
||||
@@ -224,32 +220,3 @@ function finishRetiredCronTaskRuns<T extends CronTaskRunFinalizationOutcome>(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function clearUnstartedStartupCatchupReservationMarkers(
|
||||
state: CronServiceState,
|
||||
plan: StartupCatchupReservationPlan,
|
||||
outcomes: readonly CronTaskRunFinalizationOutcome[],
|
||||
): Array<{ jobId: string; reservationIdentity: object }> {
|
||||
const pendingReleases: Array<{ jobId: string; reservationIdentity: object }> = [];
|
||||
const startedJobIds = new Set(outcomes.map((outcome) => outcome.jobId));
|
||||
for (const candidate of plan.candidates) {
|
||||
if (startedJobIds.has(candidate.jobId)) {
|
||||
continue;
|
||||
}
|
||||
const job = state.store?.jobs.find((entry) => entry.id === candidate.jobId);
|
||||
if (
|
||||
job &&
|
||||
clearQueuedCronRunReservationMarker(
|
||||
state,
|
||||
candidate.jobId,
|
||||
candidate.reservationIdentity,
|
||||
job.state,
|
||||
)
|
||||
) {
|
||||
pendingReleases.push(candidate);
|
||||
} else {
|
||||
releaseQueuedCronRun(state, candidate.jobId, candidate.reservationIdentity);
|
||||
}
|
||||
}
|
||||
return pendingReleases;
|
||||
}
|
||||
|
||||
+127
-107
@@ -5,7 +5,6 @@ import { normalizeCronRunDiagnostics, summarizeCronRunDiagnostics } from "../run
|
||||
import { resolveCronRunErrorReason } from "../run-error-reason.js";
|
||||
import { cronSchedulingInputsEqual } from "../schedule-identity.js";
|
||||
import { computeNextRunAtMs } from "../schedule.js";
|
||||
import { createCronStreamSourceIdentity } from "../stream-schedule.js";
|
||||
import type { CronJob, CronRunStatus } from "../types.js";
|
||||
import { maybeAutoDisableCronJobAfterRunFailure } from "./auto-disable.js";
|
||||
import {
|
||||
@@ -19,7 +18,7 @@ import {
|
||||
errorBackoffMs,
|
||||
isJobEnabled,
|
||||
recordScheduleComputeError,
|
||||
} from "./jobs.js";
|
||||
} from "./jobs-scheduling.js";
|
||||
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
|
||||
import { tryFinishCronTaskRun, tryFinishCronTaskRunWithoutHistory } from "./task-runs.js";
|
||||
import {
|
||||
@@ -29,9 +28,12 @@ import {
|
||||
type TimedCronRunOutcome,
|
||||
} from "./timer-execution-timeout.js";
|
||||
import {
|
||||
applyTriggerEvaluationState,
|
||||
applyTriggerRunResult,
|
||||
resolveCronNextRunWithLowerBound,
|
||||
resolveDeliveryState,
|
||||
resolveDisabledHeartbeatOneShotRetryDecision,
|
||||
resolveNextRunAtMsOrDisable,
|
||||
resolveTransientCronRetryDecision,
|
||||
shouldRetryDisabledHeartbeatOneShot,
|
||||
} from "./timer-trigger.js";
|
||||
@@ -64,6 +66,14 @@ export function resolveCronRunTriggerOwnership(params: {
|
||||
: "current";
|
||||
}
|
||||
|
||||
function assignNextRunAtMs(
|
||||
params: Parameters<typeof resolveNextRunAtMsOrDisable>[0],
|
||||
): number | undefined {
|
||||
const nextRunAtMs = resolveNextRunAtMsOrDisable(params);
|
||||
params.job.state.nextRunAtMs = nextRunAtMs;
|
||||
return nextRunAtMs;
|
||||
}
|
||||
|
||||
/** Applies run outcome state, delivery state, backoff/next-run scheduling, and delete-after-run policy. */
|
||||
export function applyJobResult(
|
||||
state: CronServiceState,
|
||||
@@ -219,17 +229,25 @@ export function applyJobResult(
|
||||
});
|
||||
if (retryDecision.retryable && retryDecision.backoffMs !== undefined) {
|
||||
job.enabled = true;
|
||||
job.state.nextRunAtMs = result.endedAt + retryDecision.backoffMs;
|
||||
state.deps.log.info(
|
||||
{
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
consecutiveSkipped: retryDecision.consecutiveSkipped,
|
||||
backoffMs: retryDecision.backoffMs,
|
||||
nextRunAtMs: job.state.nextRunAtMs,
|
||||
},
|
||||
"cron: scheduling one-shot retry after disabled heartbeat",
|
||||
);
|
||||
if (
|
||||
assignNextRunAtMs({
|
||||
state,
|
||||
job,
|
||||
candidate: result.endedAt + retryDecision.backoffMs,
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
}) !== undefined
|
||||
) {
|
||||
state.deps.log.info(
|
||||
{
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
consecutiveSkipped: retryDecision.consecutiveSkipped,
|
||||
backoffMs: retryDecision.backoffMs,
|
||||
nextRunAtMs: job.state.nextRunAtMs,
|
||||
},
|
||||
"cron: scheduling one-shot retry after disabled heartbeat",
|
||||
);
|
||||
}
|
||||
} else {
|
||||
job.enabled = false;
|
||||
job.state.nextRunAtMs = undefined;
|
||||
@@ -258,18 +276,26 @@ export function applyJobResult(
|
||||
});
|
||||
if (retryDecision.retryable && retryDecision.backoffMs !== undefined) {
|
||||
// Schedule retry with backoff (#24355).
|
||||
job.state.nextRunAtMs = result.endedAt + retryDecision.backoffMs;
|
||||
state.deps.log.info(
|
||||
{
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
consecutiveErrors: retryDecision.consecutiveErrors,
|
||||
backoffMs: retryDecision.backoffMs,
|
||||
nextRunAtMs: job.state.nextRunAtMs,
|
||||
retryCategory: retryDecision.retryCategory,
|
||||
},
|
||||
"cron: scheduling one-shot retry after transient error",
|
||||
);
|
||||
if (
|
||||
assignNextRunAtMs({
|
||||
state,
|
||||
job,
|
||||
candidate: result.endedAt + retryDecision.backoffMs,
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
}) !== undefined
|
||||
) {
|
||||
state.deps.log.info(
|
||||
{
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
consecutiveErrors: retryDecision.consecutiveErrors,
|
||||
backoffMs: retryDecision.backoffMs,
|
||||
nextRunAtMs: job.state.nextRunAtMs,
|
||||
retryCategory: retryDecision.retryCategory,
|
||||
},
|
||||
"cron: scheduling one-shot retry after transient error",
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Permanent error or max retries exhausted: disable.
|
||||
// Note: deleteAfterRun:true only triggers on ok (see shouldDelete above),
|
||||
@@ -354,25 +380,34 @@ export function applyJobResult(
|
||||
};
|
||||
if (retryDecision.retryable && retryDecision.backoffMs !== undefined) {
|
||||
normalNext = computeNormalNext();
|
||||
const retryNextRunAtMs = result.endedAt + retryDecision.backoffMs;
|
||||
if (normalNext === undefined) {
|
||||
// Preserve the unresolved-cron guard (#66019): do not synthesize a
|
||||
// retry when the schedule cannot produce a next scheduled slot.
|
||||
} else if (retryNextRunAtMs < normalNext) {
|
||||
job.state.nextRunAtMs = retryNextRunAtMs;
|
||||
state.deps.log.info(
|
||||
{
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
consecutiveErrors: retryDecision.consecutiveErrors,
|
||||
backoffMs: retryDecision.backoffMs,
|
||||
nextRunAtMs: job.state.nextRunAtMs,
|
||||
normalNextRunAtMs: normalNext,
|
||||
retryCategory: retryDecision.retryCategory,
|
||||
},
|
||||
"cron: scheduling recurring retry after transient error",
|
||||
);
|
||||
return shouldDelete;
|
||||
} else {
|
||||
const retryNextRunAtMs = assignNextRunAtMs({
|
||||
state,
|
||||
job,
|
||||
candidate: result.endedAt + retryDecision.backoffMs,
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
});
|
||||
if (retryNextRunAtMs === undefined) {
|
||||
return shouldDelete;
|
||||
}
|
||||
if (retryNextRunAtMs < normalNext) {
|
||||
state.deps.log.info(
|
||||
{
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
consecutiveErrors: retryDecision.consecutiveErrors,
|
||||
backoffMs: retryDecision.backoffMs,
|
||||
nextRunAtMs: job.state.nextRunAtMs,
|
||||
normalNextRunAtMs: normalNext,
|
||||
retryCategory: retryDecision.retryCategory,
|
||||
},
|
||||
"cron: scheduling recurring retry after transient error",
|
||||
);
|
||||
return shouldDelete;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Apply exponential backoff for errored jobs to prevent retry storms.
|
||||
@@ -381,7 +416,24 @@ export function applyJobResult(
|
||||
DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
|
||||
);
|
||||
normalNext = computeNormalNext();
|
||||
const backoffNext = result.endedAt + backoff;
|
||||
if (normalNext === undefined && job.schedule.kind === "every") {
|
||||
assignNextRunAtMs({
|
||||
state,
|
||||
job,
|
||||
candidate: undefined,
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
});
|
||||
return shouldDelete;
|
||||
}
|
||||
const backoffNext = assignNextRunAtMs({
|
||||
state,
|
||||
job,
|
||||
candidate: result.endedAt + backoff,
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
});
|
||||
if (backoffNext === undefined) {
|
||||
return shouldDelete;
|
||||
}
|
||||
// Use whichever is later: the natural next run or the backoff delay.
|
||||
job.state.nextRunAtMs =
|
||||
job.schedule.kind === "cron"
|
||||
@@ -390,7 +442,7 @@ export function applyJobResult(
|
||||
job,
|
||||
naturalNext: normalNext,
|
||||
lowerBoundMs: backoffNext,
|
||||
context: "error_backoff",
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
})
|
||||
: normalNext !== undefined
|
||||
? Math.max(normalNext, backoffNext)
|
||||
@@ -419,13 +471,17 @@ export function applyJobResult(
|
||||
});
|
||||
// The operator trigger floor is a safety policy and outranks a job-local
|
||||
// pacing bound. Non-trigger jobs retain the exact pacing clamp contract.
|
||||
const nextRunAtMs = job.trigger
|
||||
? Math.max(
|
||||
pacedNextRunAtMs,
|
||||
result.endedAt + Math.max(MIN_REFIRE_GAP_MS, resolveCronTriggerMinIntervalMs()),
|
||||
)
|
||||
: pacedNextRunAtMs;
|
||||
job.state.nextRunAtMs = nextRunAtMs;
|
||||
const nextRunAtMs = assignNextRunAtMs({
|
||||
state,
|
||||
job,
|
||||
candidate: job.trigger
|
||||
? Math.max(
|
||||
pacedNextRunAtMs ?? Number.NaN,
|
||||
result.endedAt + Math.max(MIN_REFIRE_GAP_MS, resolveCronTriggerMinIntervalMs()),
|
||||
)
|
||||
: pacedNextRunAtMs,
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
});
|
||||
job.state.pacedNextRunAtMs = nextRunAtMs;
|
||||
} else if (isJobEnabled(job)) {
|
||||
let naturalNext: number | undefined;
|
||||
@@ -460,13 +516,22 @@ export function applyJobResult(
|
||||
job,
|
||||
naturalNext,
|
||||
lowerBoundMs: minNext,
|
||||
context: "completion",
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
});
|
||||
} else {
|
||||
job.state.nextRunAtMs =
|
||||
const triggerNext =
|
||||
naturalNext !== undefined && job.trigger
|
||||
? Math.max(naturalNext, result.endedAt + resolveCronTriggerMinIntervalMs())
|
||||
: naturalNext;
|
||||
job.state.nextRunAtMs = triggerNext;
|
||||
if (triggerNext !== undefined || job.schedule.kind === "every") {
|
||||
assignNextRunAtMs({
|
||||
state,
|
||||
job,
|
||||
candidate: triggerNext,
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
job.state.nextRunAtMs = undefined;
|
||||
@@ -476,59 +541,6 @@ export function applyJobResult(
|
||||
return shouldDelete;
|
||||
}
|
||||
|
||||
function applyTriggerEvaluationState(
|
||||
job: CronJob,
|
||||
triggerEval: CronTriggerEvalOutcome,
|
||||
evaluatedAtMs: number,
|
||||
): void {
|
||||
if (triggerEval.busy) {
|
||||
return;
|
||||
}
|
||||
job.state.lastTriggerEvalAtMs = evaluatedAtMs;
|
||||
job.state.triggerEvalCount = (job.state.triggerEvalCount ?? 0) + 1;
|
||||
if (triggerEval.stateChanged) {
|
||||
job.state.triggerState = triggerEval.state;
|
||||
}
|
||||
if (triggerEval.fired) {
|
||||
job.state.lastTriggerFireAtMs = evaluatedAtMs;
|
||||
}
|
||||
}
|
||||
|
||||
/** Persists fired/error evaluation metadata and applies successful once-disarm policy. */
|
||||
export function applyTriggerRunResult(
|
||||
job: CronJob,
|
||||
result: { status: CronRunStatus; endedAt: number; triggerEval?: CronTriggerEvalOutcome },
|
||||
opts?: { scheduleOwnership?: CronScheduleOwnership; triggerOwnership?: CronTriggerOwnership },
|
||||
): void {
|
||||
if (!result.triggerEval || opts?.triggerOwnership === "stale") {
|
||||
return;
|
||||
}
|
||||
// Fired-run trigger state persists only on payload success: a failed or
|
||||
// skipped run keeps the previous state so the next evaluation re-detects
|
||||
// the change and fires again instead of silently losing the event.
|
||||
const persistedEval =
|
||||
result.status === "ok"
|
||||
? result.triggerEval
|
||||
: { ...result.triggerEval, stateChanged: false, state: undefined };
|
||||
applyTriggerEvaluationState(job, persistedEval, result.endedAt);
|
||||
// A once trigger disarms only after the fired payload succeeds. Errors keep
|
||||
// it armed so the normal backoff path can evaluate and retry later.
|
||||
if (
|
||||
opts?.scheduleOwnership !== "stale" &&
|
||||
result.triggerEval.fired &&
|
||||
job.trigger?.once === true &&
|
||||
result.status === "ok"
|
||||
) {
|
||||
if (job.schedule.kind === "stream") {
|
||||
// Auto-disable is a source retirement just like an explicit disable. Rotate
|
||||
// in the same persisted result so queued sibling batches cannot gain admission.
|
||||
job.state.streamSourceIdentity = createCronStreamSourceIdentity();
|
||||
}
|
||||
job.enabled = false;
|
||||
job.state.nextRunAtMs = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Commits payload-script state only after the complete cron run succeeds. */
|
||||
export function applyScriptRunResult(
|
||||
job: CronJob,
|
||||
@@ -591,8 +603,16 @@ export function applyTriggerNoFireResult(
|
||||
const floorMs = Math.max(MIN_REFIRE_GAP_MS, resolveCronTriggerMinIntervalMs());
|
||||
// Quiet ticks still advance the schedule; the floor prevents scripts from
|
||||
// becoming a headless hot loop even when cron resolves inside the window.
|
||||
job.state.nextRunAtMs =
|
||||
naturalNext === undefined ? undefined : Math.max(naturalNext, result.endedAt + floorMs);
|
||||
job.state.nextRunAtMs = naturalNext;
|
||||
if (naturalNext !== undefined || job.schedule.kind === "every") {
|
||||
assignNextRunAtMs({
|
||||
state,
|
||||
job,
|
||||
candidate:
|
||||
naturalNext === undefined ? undefined : Math.max(naturalNext, result.endedAt + floorMs),
|
||||
deferredNotifications: opts?.deferredNotifications,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
recordScheduleComputeError({
|
||||
state,
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
isJobEnabled,
|
||||
resolveJobErrorBackoffUntilMs,
|
||||
resolveJobLastRunStatus,
|
||||
} from "./jobs.js";
|
||||
} from "./jobs-scheduling.js";
|
||||
import type { CronServiceState } from "./state.js";
|
||||
import { isScheduledTerminalOneShotRetry } from "./timer-trigger.js";
|
||||
|
||||
|
||||
@@ -5,46 +5,35 @@ import {
|
||||
GatewayDrainingError,
|
||||
} from "../../process/gateway-work-admission.js";
|
||||
import { normalizeAgentId, resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
|
||||
import { markCronJobActive } from "../active-jobs.js";
|
||||
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
|
||||
import { sweepCronRunSessions } from "../session-reaper.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import {
|
||||
hasScheduledNextRunAtMs,
|
||||
isJobEnabled,
|
||||
nextWakeAtMs,
|
||||
recomputeNextRunsForMaintenance,
|
||||
} from "./jobs.js";
|
||||
} from "./jobs-scheduling.js";
|
||||
import { locked } from "./locked.js";
|
||||
import {
|
||||
activateQueuedCronRun,
|
||||
clearQueuedCronRunReservationMarker,
|
||||
isQueuedCronRunReservationCurrent,
|
||||
isQueuedCronRunReservationMarkerCurrent,
|
||||
cleanupQueuedCronRunReservations,
|
||||
executeQueuedCronRun,
|
||||
releaseQueuedCronRun,
|
||||
reserveQueuedCronRun,
|
||||
resolveRunConcurrency,
|
||||
restoreQueuedCronRunReservationLastError,
|
||||
runWithCronAdmission,
|
||||
} from "./run-admission.js";
|
||||
import { type CronServiceState, type DeferredCronNotifications, emit } from "./state.js";
|
||||
import type { CronServiceState, DeferredCronNotifications } from "./state.js";
|
||||
import { ensureLoaded, persistOrRestore, snapshotStoreForRollback } from "./store.js";
|
||||
import { tryCreateCronTaskRun } from "./task-runs.js";
|
||||
import { resolveCronJobTimeoutMs } from "./timeout-policy.js";
|
||||
import {
|
||||
MAX_TIMER_DELAY_MS,
|
||||
MIN_REFIRE_GAP_MS,
|
||||
runsDetachedFromMainSession,
|
||||
type TimedCronRunOutcome,
|
||||
} from "./timer-execution-timeout.js";
|
||||
import { executeJobCoreWithTimeout } from "./timer-job-runner.js";
|
||||
import { maybeNotifyIsolatedAgentSetupTimeoutWithRecovery } from "./timer-notifications.js";
|
||||
import {
|
||||
createCompletedCronRunOutcomeDrain,
|
||||
finalizeCompletedCronRunOutcomes,
|
||||
} from "./timer-outcome-finalization.js";
|
||||
import { collectRunnableJobs, isRunnableJob } from "./timer-runnable.js";
|
||||
import { collectRunnableJobs } from "./timer-runnable.js";
|
||||
|
||||
export function maybeNotifyIsolatedAgentSetupTimeout(
|
||||
state: CronServiceState,
|
||||
@@ -233,130 +222,9 @@ async function onAdmittedTimer(state: CronServiceState) {
|
||||
reservedAtMs: now,
|
||||
reservationIdentity: reserveQueuedCronRun(state, job.id, now),
|
||||
}));
|
||||
if (state.stopped) {
|
||||
const cleanup = async () => {
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
const pendingReleases: typeof reservedDue = [];
|
||||
for (const candidate of reservedDue) {
|
||||
if (
|
||||
!isQueuedCronRunReservationCurrent(state, candidate.id, candidate.reservationIdentity)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const persistedJob = state.store?.jobs.find((entry) => entry.id === candidate.id);
|
||||
if (
|
||||
typeof persistedJob?.state.queuedAtMs === "number" &&
|
||||
isQueuedCronRunReservationMarkerCurrent(
|
||||
state,
|
||||
candidate.id,
|
||||
candidate.reservationIdentity,
|
||||
persistedJob.state.queuedAtMs,
|
||||
)
|
||||
) {
|
||||
restoreQueuedCronRunReservationLastError(
|
||||
state,
|
||||
candidate.id,
|
||||
candidate.reservationIdentity,
|
||||
persistedJob.state,
|
||||
);
|
||||
delete persistedJob.state.queuedAtMs;
|
||||
pendingReleases.push(candidate);
|
||||
} else {
|
||||
releaseQueuedCronRun(state, candidate.id, candidate.reservationIdentity);
|
||||
}
|
||||
}
|
||||
const postPersistNotifications: DeferredCronNotifications = [];
|
||||
recomputeNextRunsForMaintenance(state, {
|
||||
deferredNotifications: postPersistNotifications,
|
||||
});
|
||||
await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications });
|
||||
for (const candidate of pendingReleases) {
|
||||
releaseQueuedCronRun(state, candidate.id, candidate.reservationIdentity);
|
||||
}
|
||||
};
|
||||
try {
|
||||
await cleanup();
|
||||
} catch {
|
||||
try {
|
||||
await cleanup();
|
||||
} catch (error) {
|
||||
// The stopped scheduler has no later cleanup pass.
|
||||
for (const candidate of reservedDue) {
|
||||
releaseQueuedCronRun(state, candidate.id, candidate.reservationIdentity);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
return reservedDue;
|
||||
});
|
||||
|
||||
const runDueJob = async (params: {
|
||||
id: string;
|
||||
job: CronJob;
|
||||
reservationIdentity: object;
|
||||
startedAt: number;
|
||||
}): Promise<TimedCronRunOutcome> => {
|
||||
const { id, job, startedAt } = params;
|
||||
const executionJob = structuredClone(job);
|
||||
executionJob.state.runningAtMs = startedAt;
|
||||
executionJob.state.lastError = undefined;
|
||||
const activeJobMarker = markCronJobActive(executionJob.id, {
|
||||
preserveAcrossGenerationAdvance: !runsDetachedFromMainSession(executionJob),
|
||||
});
|
||||
emit(state, {
|
||||
jobId: executionJob.id,
|
||||
action: "started",
|
||||
job: executionJob,
|
||||
runAtMs: startedAt,
|
||||
});
|
||||
const jobTimeoutMs = resolveCronJobTimeoutMs(executionJob);
|
||||
const taskRunId = tryCreateCronTaskRun({
|
||||
state,
|
||||
job: executionJob,
|
||||
startedAt,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await executeJobCoreWithTimeout(state, executionJob, {
|
||||
runId: taskRunId,
|
||||
activeJobMarker,
|
||||
});
|
||||
return {
|
||||
jobId: id,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
reservationIdentity: params.reservationIdentity,
|
||||
...result,
|
||||
startedAt,
|
||||
endedAt: state.deps.nowMs(),
|
||||
};
|
||||
} catch (err) {
|
||||
const errorText = normalizeCronRunErrorText(err);
|
||||
state.deps.log.warn(
|
||||
{ jobId: id, jobName: executionJob.name, timeoutMs: jobTimeoutMs ?? null },
|
||||
`cron: job failed: ${errorText}`,
|
||||
);
|
||||
return {
|
||||
jobId: id,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
reservationIdentity: params.reservationIdentity,
|
||||
status: "error",
|
||||
error: errorText,
|
||||
diagnostics: createCronRunDiagnosticsFromError("cron-setup", errorText, {
|
||||
nowMs: state.deps.nowMs,
|
||||
}),
|
||||
startedAt,
|
||||
endedAt: state.deps.nowMs(),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const concurrency = Math.min(resolveRunConcurrency(), Math.max(1, dueJobs.length));
|
||||
const completedOutcomeDrain = createCompletedCronRunOutcomeDrain(state);
|
||||
const claimedIndexes = new Set<number>();
|
||||
@@ -364,53 +232,18 @@ async function onAdmittedTimer(state: CronServiceState) {
|
||||
let setupTimeoutNotified = false;
|
||||
let stopAdmittingDueJobs = false;
|
||||
const hasSetupTimeoutRecoveryHandler = state.deps.onIsolatedAgentSetupTimeout !== undefined;
|
||||
const releaseUnclaimedDueJobReservations = async () => {
|
||||
if (claimedIndexes.size >= dueJobs.length) {
|
||||
return;
|
||||
}
|
||||
await locked(state, async () => {
|
||||
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
const pendingReleases: typeof dueJobs = [];
|
||||
for (const [index, due] of dueJobs.entries()) {
|
||||
if (claimedIndexes.has(index)) {
|
||||
continue;
|
||||
}
|
||||
const job = state.store?.jobs.find((entry) => entry.id === due.id);
|
||||
if (
|
||||
job &&
|
||||
clearQueuedCronRunReservationMarker(state, due.id, due.reservationIdentity, job.state)
|
||||
) {
|
||||
pendingReleases.push(due);
|
||||
} else {
|
||||
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
|
||||
}
|
||||
}
|
||||
const postPersistNotifications: DeferredCronNotifications = [];
|
||||
recomputeNextRunsForMaintenance(state, { deferredNotifications: postPersistNotifications });
|
||||
await persistOrRestore(state, rollbackSnapshot, { postPersistNotifications });
|
||||
for (const due of pendingReleases) {
|
||||
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
|
||||
}
|
||||
});
|
||||
};
|
||||
const releaseUnclaimedDueJobReservationsWithRetry = async () => {
|
||||
try {
|
||||
await releaseUnclaimedDueJobReservations();
|
||||
} catch {
|
||||
try {
|
||||
await releaseUnclaimedDueJobReservations();
|
||||
} catch (error) {
|
||||
// No timer task owns another retry. Drop only these process claims so
|
||||
// durable stuck-marker recovery remains able to repair them.
|
||||
for (const [index, due] of dueJobs.entries()) {
|
||||
if (!claimedIndexes.has(index)) {
|
||||
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const reservations = dueJobs
|
||||
.filter((_, index) => !claimedIndexes.has(index))
|
||||
.map((due) => ({
|
||||
jobId: due.id,
|
||||
reservationIdentity: due.reservationIdentity,
|
||||
}));
|
||||
await cleanupQueuedCronRunReservations({
|
||||
state,
|
||||
reservations,
|
||||
recompute: "maintenance",
|
||||
});
|
||||
};
|
||||
if (state.stopped) {
|
||||
await releaseUnclaimedDueJobReservationsWithRetry();
|
||||
@@ -429,92 +262,75 @@ async function onAdmittedTimer(state: CronServiceState) {
|
||||
return pMapSkip;
|
||||
}
|
||||
try {
|
||||
const admission = await runWithCronAdmission(state, async () => {
|
||||
const currentDueJob = await locked(state, async () => {
|
||||
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
|
||||
if (stopAdmittingDueJobs || state.stopped || state.restartRecoveryPending) {
|
||||
stopAdmittingDueJobs = true;
|
||||
return undefined;
|
||||
}
|
||||
const job = state.store?.jobs.find((entry) => entry.id === due.id);
|
||||
if (
|
||||
!job ||
|
||||
!isQueuedCronRunReservationCurrent(state, due.id, due.reservationIdentity) ||
|
||||
job.state.queuedAtMs !== due.reservedAtMs
|
||||
) {
|
||||
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
|
||||
return undefined;
|
||||
}
|
||||
const dueProbe = structuredClone(job);
|
||||
delete dueProbe.state.queuedAtMs;
|
||||
if (
|
||||
!isJobEnabled(job) ||
|
||||
!isRunnableJob({ state, job: dueProbe, nowMs: state.deps.nowMs() })
|
||||
) {
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
delete job.state.queuedAtMs;
|
||||
await persistOrRestore(state, rollbackSnapshot);
|
||||
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
|
||||
return undefined;
|
||||
}
|
||||
const activation = await activateQueuedCronRun({
|
||||
state,
|
||||
job,
|
||||
reservationIdentity: due.reservationIdentity,
|
||||
onUnavailable: () => {
|
||||
stopAdmittingDueJobs = true;
|
||||
},
|
||||
});
|
||||
if (activation.kind === "unavailable") {
|
||||
return undefined;
|
||||
}
|
||||
return { ...due, job, startedAt: activation.startedAt };
|
||||
});
|
||||
if (!currentDueJob) {
|
||||
return pMapSkip;
|
||||
}
|
||||
claimedIndexes.add(index);
|
||||
let result: TimedCronRunOutcome;
|
||||
try {
|
||||
result = await runDueJob(currentDueJob);
|
||||
} catch (error) {
|
||||
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
|
||||
throw error;
|
||||
}
|
||||
if (!result.isolatedAgentSetupTimeout) {
|
||||
// Drain finished state independently: a slow sibling must not
|
||||
// strand outcomes, and store I/O must not own execution slots.
|
||||
completedOutcomeDrain.enqueue(result);
|
||||
return pMapSkip;
|
||||
}
|
||||
let finalizedResults: TimedCronRunOutcome[];
|
||||
try {
|
||||
finalizedResults = await finalizeCompletedCronRunOutcomes(state, [result], {
|
||||
clearOnFailure: false,
|
||||
});
|
||||
} catch {
|
||||
return result;
|
||||
}
|
||||
if (!hasSetupTimeoutRecoveryHandler || finalizedResults.length === 0) {
|
||||
return pMapSkip;
|
||||
}
|
||||
if (!setupTimeoutNotified) {
|
||||
setupTimeoutNotified = true;
|
||||
const execution = await executeQueuedCronRun({
|
||||
state,
|
||||
jobId: due.id,
|
||||
reservedAtMs: due.reservedAtMs,
|
||||
reservationIdentity: due.reservationIdentity,
|
||||
isUnavailable: () => stopAdmittingDueJobs,
|
||||
onUnavailable: () => {
|
||||
stopAdmittingDueJobs = true;
|
||||
try {
|
||||
await releaseUnclaimedDueJobReservationsWithRetry();
|
||||
} catch (err) {
|
||||
reservationReleaseError = err;
|
||||
},
|
||||
onActivated: () => claimedIndexes.add(index),
|
||||
onNotRunnable: async (job) => {
|
||||
const rollbackSnapshot = snapshotStoreForRollback(state);
|
||||
delete job.state.queuedAtMs;
|
||||
await persistOrRestore(state, rollbackSnapshot);
|
||||
releaseQueuedCronRun(state, due.id, due.reservationIdentity);
|
||||
},
|
||||
onSetupError: (job, errorText) => {
|
||||
state.deps.log.warn(
|
||||
{
|
||||
jobId: due.id,
|
||||
jobName: job.name,
|
||||
timeoutMs: resolveCronJobTimeoutMs(job) ?? null,
|
||||
},
|
||||
`cron: job failed: ${errorText}`,
|
||||
);
|
||||
},
|
||||
onCompleted: async (result) => {
|
||||
if (!result.isolatedAgentSetupTimeout) {
|
||||
// Drain finished state independently: a slow sibling must not
|
||||
// strand outcomes, and store I/O must not own execution slots.
|
||||
completedOutcomeDrain.enqueue(result);
|
||||
return true;
|
||||
}
|
||||
maybeNotifyIsolatedAgentSetupTimeout(state, result);
|
||||
}
|
||||
return pMapSkip;
|
||||
let finalizedResults: TimedCronRunOutcome[];
|
||||
try {
|
||||
finalizedResults = await finalizeCompletedCronRunOutcomes(state, [result], {
|
||||
clearOnFailure: false,
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
hasSetupTimeoutRecoveryHandler &&
|
||||
finalizedResults.length > 0 &&
|
||||
!setupTimeoutNotified
|
||||
) {
|
||||
setupTimeoutNotified = true;
|
||||
stopAdmittingDueJobs = true;
|
||||
try {
|
||||
await releaseUnclaimedDueJobReservationsWithRetry();
|
||||
} catch (err) {
|
||||
reservationReleaseError = err;
|
||||
}
|
||||
maybeNotifyIsolatedAgentSetupTimeout(state, result);
|
||||
}
|
||||
return true;
|
||||
},
|
||||
});
|
||||
if (admission.kind === "stopped") {
|
||||
if (execution.kind === "stopped") {
|
||||
stopAdmittingDueJobs = true;
|
||||
return pMapSkip;
|
||||
}
|
||||
return admission.value;
|
||||
if (execution.kind === "skipped") {
|
||||
return pMapSkip;
|
||||
}
|
||||
if (execution.handled) {
|
||||
return pMapSkip;
|
||||
}
|
||||
return execution.outcome;
|
||||
} catch (error) {
|
||||
stopAdmittingDueJobs = true;
|
||||
batchExecutionError ??= error;
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { asDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
|
||||
import type { CronConfig } from "../../config/types.cron.js";
|
||||
import { resolveCronDeliveryPlan, resolveFailureDestination } from "../delivery-plan.js";
|
||||
import { type CronRetryOn, resolveCronExecutionRetryHint } from "../retry-hint.js";
|
||||
import { createCronStreamSourceIdentity } from "../stream-schedule.js";
|
||||
import type {
|
||||
CronDeliveryStatus,
|
||||
CronFailureNotificationDelivery,
|
||||
@@ -8,8 +10,18 @@ import type {
|
||||
CronRunErrorClassification,
|
||||
CronRunStatus,
|
||||
} from "../types.js";
|
||||
import { DEFAULT_ERROR_BACKOFF_SCHEDULE_MS, errorBackoffMs, isJobEnabled } from "./jobs.js";
|
||||
import type { CronServiceState, CronSystemEventEnqueueResult } from "./state.js";
|
||||
import { autoDisableCronJob } from "./auto-disable.js";
|
||||
import {
|
||||
DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
|
||||
errorBackoffMs,
|
||||
isJobEnabled,
|
||||
} from "./jobs-scheduling.js";
|
||||
import type {
|
||||
CronServiceState,
|
||||
CronSystemEventEnqueueResult,
|
||||
DeferredCronNotifications,
|
||||
} from "./state.js";
|
||||
import type { CronTriggerEvalOutcome } from "./timer-execution-timeout.js";
|
||||
import { HEARTBEAT_SKIP_DISABLED } from "./timer-execution-timeout.js";
|
||||
|
||||
/** Default max retries for cron jobs on transient errors (#24355). */
|
||||
@@ -35,25 +47,100 @@ type QueuedSystemEventHandle = {
|
||||
remove?: () => boolean | void;
|
||||
};
|
||||
|
||||
/** Rejects outcome-generated schedule timestamps before they can persist or arm a timer. */
|
||||
export function resolveNextRunAtMsOrDisable(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
candidate: unknown;
|
||||
deferredNotifications?: DeferredCronNotifications;
|
||||
}): number | undefined {
|
||||
const nextRunAtMs = asDateTimestampMs(params.candidate);
|
||||
if (nextRunAtMs !== undefined && nextRunAtMs > 0) {
|
||||
return nextRunAtMs;
|
||||
}
|
||||
autoDisableCronJob({
|
||||
state: params.state,
|
||||
job: params.job,
|
||||
reason: "schedule-errors",
|
||||
atMs: params.state.deps.nowMs(),
|
||||
consecutiveErrors: 1,
|
||||
error: "next run is outside the supported Date range",
|
||||
deferredNotifications: params.deferredNotifications,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Persists non-busy trigger evaluation state without touching payload-run history. */
|
||||
export function applyTriggerEvaluationState(
|
||||
job: CronJob,
|
||||
triggerEval: CronTriggerEvalOutcome,
|
||||
evaluatedAtMs: number,
|
||||
): void {
|
||||
if (triggerEval.busy) {
|
||||
return;
|
||||
}
|
||||
job.state.lastTriggerEvalAtMs = evaluatedAtMs;
|
||||
job.state.triggerEvalCount = (job.state.triggerEvalCount ?? 0) + 1;
|
||||
if (triggerEval.stateChanged) {
|
||||
job.state.triggerState = triggerEval.state;
|
||||
}
|
||||
if (triggerEval.fired) {
|
||||
job.state.lastTriggerFireAtMs = evaluatedAtMs;
|
||||
}
|
||||
}
|
||||
|
||||
/** Persists fired/error trigger metadata and disarms successful once triggers. */
|
||||
export function applyTriggerRunResult(
|
||||
job: CronJob,
|
||||
result: { status: CronRunStatus; endedAt: number; triggerEval?: CronTriggerEvalOutcome },
|
||||
opts?: { scheduleOwnership?: "current" | "stale"; triggerOwnership?: "current" | "stale" },
|
||||
): void {
|
||||
if (!result.triggerEval || opts?.triggerOwnership === "stale") {
|
||||
return;
|
||||
}
|
||||
// Failed payloads keep the old state so the next evaluation re-detects the event.
|
||||
const persistedEval =
|
||||
result.status === "ok"
|
||||
? result.triggerEval
|
||||
: { ...result.triggerEval, stateChanged: false, state: undefined };
|
||||
applyTriggerEvaluationState(job, persistedEval, result.endedAt);
|
||||
if (
|
||||
opts?.scheduleOwnership !== "stale" &&
|
||||
result.triggerEval.fired &&
|
||||
job.trigger?.once === true &&
|
||||
result.status === "ok"
|
||||
) {
|
||||
if (job.schedule.kind === "stream") {
|
||||
job.state.streamSourceIdentity = createCronStreamSourceIdentity();
|
||||
}
|
||||
job.enabled = false;
|
||||
job.state.nextRunAtMs = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveCronNextRunWithLowerBound(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
naturalNext: number | undefined;
|
||||
lowerBoundMs: number;
|
||||
context: "completion" | "error_backoff";
|
||||
deferredNotifications?: DeferredCronNotifications;
|
||||
}): number | undefined {
|
||||
if (params.naturalNext === undefined) {
|
||||
params.state.deps.log.warn(
|
||||
{
|
||||
jobId: params.job.id,
|
||||
jobName: params.job.name,
|
||||
context: params.context,
|
||||
},
|
||||
"cron: next run unresolved; clearing schedule to avoid a refire loop",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
return Math.max(params.naturalNext, params.lowerBoundMs);
|
||||
return resolveNextRunAtMsOrDisable({
|
||||
state: params.state,
|
||||
job: params.job,
|
||||
candidate: Math.max(params.naturalNext, params.lowerBoundMs),
|
||||
deferredNotifications: params.deferredNotifications,
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveTransientCronRetryDecision(params: {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Completed cron work must become durable before unrelated batch work drains.
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createDueIsolatedJob,
|
||||
@@ -427,6 +428,84 @@ describe("cron batch outcome finalization", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("records and notifies a Date-overflow auto-disable only after persistence", async () => {
|
||||
const store = fixtures.makeStorePath();
|
||||
const dueAt = Date.parse("2026-08-01T15:02:00.000Z");
|
||||
const job = createDueIsolatedJob({
|
||||
id: "date-overflow-auto-disable",
|
||||
nowMs: dueAt,
|
||||
nextRunAtMs: dueAt,
|
||||
});
|
||||
job.schedule = { kind: "every", everyMs: 60_000, anchorMs: dueAt - 60_000 };
|
||||
job.pacing = { min: "1s" };
|
||||
job.state.runningAtMs = dueAt;
|
||||
await saveCronStore(store.storePath, { version: 1, jobs: [job] });
|
||||
|
||||
const order: string[] = [];
|
||||
const enqueueSystemEvent = vi.fn((_text: string) => {
|
||||
order.push("notify");
|
||||
});
|
||||
const requestHeartbeat = vi.fn(() => {
|
||||
order.push("heartbeat");
|
||||
});
|
||||
const state = createCronServiceState({
|
||||
cronEnabled: true,
|
||||
storePath: store.storePath,
|
||||
log: noopLogger,
|
||||
nowMs: () => dueAt + 10,
|
||||
enqueueSystemEvent,
|
||||
requestHeartbeat,
|
||||
runIsolatedAgentJob: vi.fn(),
|
||||
});
|
||||
const save = cronStoreModule.saveCronJobsStore;
|
||||
const saveSpy = vi
|
||||
.spyOn(cronStoreModule, "saveCronJobsStore")
|
||||
.mockImplementation(async (...args) => {
|
||||
if (args[1].jobs[0]?.state.autoDisabled) {
|
||||
expect(enqueueSystemEvent).not.toHaveBeenCalled();
|
||||
expect(requestHeartbeat).not.toHaveBeenCalled();
|
||||
order.push("persist");
|
||||
}
|
||||
return await save(...args);
|
||||
});
|
||||
|
||||
try {
|
||||
const finalized = await finalizeCompletedCronRunOutcomes(state, [
|
||||
{
|
||||
jobId: job.id,
|
||||
job: structuredClone(job),
|
||||
activeJobMarker: markCronJobActive(job.id),
|
||||
status: "ok",
|
||||
startedAt: dueAt,
|
||||
endedAt: dueAt + 10,
|
||||
nextCheck: { delayMs: MAX_DATE_TIMESTAMP_MS },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(finalized).toHaveLength(1);
|
||||
expect(state.store?.jobs[0]?.enabled).toBe(false);
|
||||
expect(state.store?.jobs[0]?.state.nextRunAtMs).toBeUndefined();
|
||||
expect(order).toEqual(["persist", "notify", "heartbeat"]);
|
||||
expect(enqueueSystemEvent).toHaveBeenCalledOnce();
|
||||
expect(enqueueSystemEvent.mock.calls[0]?.[0]).toContain(
|
||||
"next run is outside the supported Date range",
|
||||
);
|
||||
expect(requestHeartbeat).toHaveBeenCalledOnce();
|
||||
expect((await loadCronStore(store.storePath)).jobs[0]).toMatchObject({
|
||||
enabled: false,
|
||||
state: {
|
||||
autoDisabled: {
|
||||
reason: "schedule-errors",
|
||||
atMs: dueAt + 10,
|
||||
consecutiveErrors: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
saveSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("rolls back recurring auto-disable without notifying when persistence fails", async () => {
|
||||
const store = fixtures.makeStorePath();
|
||||
const dueAt = Date.parse("2026-08-01T15:05:00.000Z");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
clearCronJobActive,
|
||||
@@ -7,7 +8,7 @@ import {
|
||||
import { makeCronJob } from "../delivery.test-helpers.js";
|
||||
import { createNoopLogger } from "../service.test-harness.js";
|
||||
import type { CronJob, CronPacing } from "../types.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs.js";
|
||||
import { recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { createCronServiceState } from "./state.js";
|
||||
import { applyOutcomeToStoredJob, applyTriggerNoFireResult } from "./timer-outcomes.js";
|
||||
import { applyJobResult } from "./timer.js";
|
||||
@@ -120,6 +121,95 @@ describe("cron trigger evaluation ownership", () => {
|
||||
});
|
||||
|
||||
describe("applyJobResult dynamic cadence", () => {
|
||||
it.each(["one-shot retry", "recurring retry", "pacing", "trigger floor", "quiet trigger"])(
|
||||
"auto-disables a job when %s cannot produce a Date-valid next run",
|
||||
(scenario) => {
|
||||
const endedAt = MAX_DATE_TIMESTAMP_MS - 1_000;
|
||||
const state = makeState();
|
||||
const deferredNotifications: Array<() => void> = [];
|
||||
const job = makeCronJob({
|
||||
schedule:
|
||||
scenario === "one-shot retry"
|
||||
? { kind: "at", at: new Date(endedAt).toISOString() }
|
||||
: { kind: "every", everyMs: 1_000, anchorMs: 0 },
|
||||
state: { nextRunAtMs: endedAt },
|
||||
...(scenario === "pacing" ? { pacing: { min: "1s" } } : {}),
|
||||
...(scenario === "trigger floor" || scenario === "quiet trigger"
|
||||
? { trigger: { script: "return true" } }
|
||||
: {}),
|
||||
});
|
||||
|
||||
if (scenario === "quiet trigger") {
|
||||
applyTriggerNoFireResult(
|
||||
state,
|
||||
job,
|
||||
{
|
||||
startedAt: endedAt - 1,
|
||||
endedAt,
|
||||
triggerEval: { fired: false, stateChanged: false },
|
||||
},
|
||||
{ deferredNotifications },
|
||||
);
|
||||
} else {
|
||||
const isRetry = scenario === "one-shot retry" || scenario === "recurring retry";
|
||||
applyJobResult(
|
||||
state,
|
||||
job,
|
||||
{
|
||||
status: isRetry ? "error" : "ok",
|
||||
...(isRetry
|
||||
? {
|
||||
error: "temporary timeout",
|
||||
errorClassification: { kind: "reason" as const, reason: "timeout" as const },
|
||||
executionStarted: true,
|
||||
}
|
||||
: {}),
|
||||
startedAt: endedAt - 1,
|
||||
endedAt,
|
||||
...(scenario === "pacing" ? { nextCheck: { delayMs: 2_000 } } : {}),
|
||||
},
|
||||
{ deferredNotifications },
|
||||
);
|
||||
}
|
||||
|
||||
expect(job.enabled).toBe(false);
|
||||
expect(job.state.nextRunAtMs).toBeUndefined();
|
||||
expect(job.state.pacedNextRunAtMs).toBeUndefined();
|
||||
expect(job.state.autoDisabled).toEqual({
|
||||
reason: "schedule-errors",
|
||||
atMs: ENDED_AT,
|
||||
consecutiveErrors: 1,
|
||||
});
|
||||
expect(state.deps.enqueueSystemEvent).not.toHaveBeenCalled();
|
||||
expect(state.deps.requestHeartbeat).not.toHaveBeenCalled();
|
||||
expect(deferredNotifications).toHaveLength(1);
|
||||
|
||||
deferredNotifications[0]?.();
|
||||
expect(state.deps.enqueueSystemEvent).toHaveBeenCalledOnce();
|
||||
expect(state.deps.requestHeartbeat).toHaveBeenCalledOnce();
|
||||
},
|
||||
);
|
||||
|
||||
it("disables an exhausted every schedule instead of synthesizing a backoff-only run", () => {
|
||||
const endedAt = MAX_DATE_TIMESTAMP_MS - 39_000;
|
||||
const job = makeCronJob({
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 0 },
|
||||
state: { nextRunAtMs: endedAt },
|
||||
});
|
||||
|
||||
applyJobResult(makeState(), job, {
|
||||
status: "error",
|
||||
error: "permanent failure",
|
||||
errorClassification: { kind: "permanent" },
|
||||
startedAt: endedAt - 1_000,
|
||||
endedAt,
|
||||
});
|
||||
|
||||
expect(endedAt + 30_000).toBeLessThanOrEqual(MAX_DATE_TIMESTAMP_MS);
|
||||
expect(job.enabled).toBe(false);
|
||||
expect(job.state.nextRunAtMs).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["honors an in-range proposal", { min: "15m", max: "4h" }, 60 * 60_000, 60 * 60_000],
|
||||
["clamps below the minimum", { min: "15m", max: "4h" }, 5 * 60_000, 15 * 60_000],
|
||||
|
||||
@@ -33,7 +33,7 @@ import { readCronTaskRunHistoryPage } from "../task-run-history.js";
|
||||
import type { CronAgentExecutionPhaseUpdate, CronJob } from "../types.js";
|
||||
import { cancelActiveCronTaskRun } from "./active-run-cancellation.js";
|
||||
import { resetActiveCronTaskRunsForTests } from "./active-run-cancellation.test-support.js";
|
||||
import { computeJobNextRunAtMs, recomputeNextRunsForMaintenance } from "./jobs.js";
|
||||
import { computeJobNextRunAtMs, recomputeNextRunsForMaintenance } from "./jobs-scheduling.js";
|
||||
import { stop } from "./ops-lifecycle.js";
|
||||
import { run as runManualCronJob } from "./ops-run.js";
|
||||
import { createCronServiceState as createBaseCronServiceState, type CronEvent } from "./state.js";
|
||||
@@ -2152,6 +2152,74 @@ describe("cron service timer regressions", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not release a running sibling when setup-timeout recovery clears queued jobs", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const store = timerRegressionFixtures.makeStorePath();
|
||||
const dueAt = Date.parse("2026-02-06T10:06:21.000Z");
|
||||
const stalled = createDueIsolatedJob({
|
||||
id: "setup-timeout-stalled",
|
||||
nowMs: dueAt,
|
||||
nextRunAtMs: dueAt,
|
||||
});
|
||||
const running = createDueIsolatedJob({
|
||||
id: "setup-timeout-running-sibling",
|
||||
nowMs: dueAt,
|
||||
nextRunAtMs: dueAt,
|
||||
});
|
||||
stalled.payload = { kind: "agentTurn", message: "stall", timeoutSeconds: 120 };
|
||||
running.payload = { kind: "agentTurn", message: "run", timeoutSeconds: 120 };
|
||||
await saveCronStore(store.storePath, { version: 1, jobs: [stalled, running] });
|
||||
|
||||
let now = dueAt;
|
||||
const runningStarted = createDeferred();
|
||||
const finishRunning = createDeferred<{ status: "ok"; summary: string }>();
|
||||
const timeoutNotified = createDeferred();
|
||||
const state = createCronServiceState({
|
||||
cronEnabled: true,
|
||||
storePath: store.storePath,
|
||||
testAdmissionLimit: 2,
|
||||
log: noopLogger,
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
onIsolatedAgentSetupTimeout: () => timeoutNotified.resolve(),
|
||||
runIsolatedAgentJob: vi.fn(
|
||||
async ({
|
||||
job,
|
||||
onExecutionStarted,
|
||||
}: Parameters<CronStateParams["runIsolatedAgentJob"]>[0]) => {
|
||||
if (job.id === stalled.id) {
|
||||
return await new Promise<never>(() => {});
|
||||
}
|
||||
onExecutionStarted?.({ jobId: job.id, phase: "model_call_started" });
|
||||
runningStarted.resolve();
|
||||
return await finishRunning.promise;
|
||||
},
|
||||
),
|
||||
});
|
||||
|
||||
const timerPromise = onTimer(state);
|
||||
await runningStarted.promise;
|
||||
await vi.advanceTimersByTimeAsync(60_100);
|
||||
now += 60_100;
|
||||
await timeoutNotified.promise;
|
||||
|
||||
const runningAtMsAfterRecovery = requireJob(state, running.id).state.runningAtMs;
|
||||
const reservationHeldAfterRecovery = state.queuedRunReservationsByJobId.has(running.id);
|
||||
finishRunning.resolve({ status: "ok", summary: "finished" });
|
||||
await timerPromise;
|
||||
|
||||
expect({ runningAtMsAfterRecovery, reservationHeldAfterRecovery }).toEqual({
|
||||
runningAtMsAfterRecovery: dueAt,
|
||||
reservationHeldAfterRecovery: true,
|
||||
});
|
||||
expect(requireJob(state, running.id).state.lastStatus).toBe("ok");
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("notifies timeout recovery before admitting queued manual work", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
|
||||
@@ -7,7 +7,7 @@ export { runsDetachedFromMainSession } from "./timer-execution-timeout.js";
|
||||
export { executeJobCoreWithTimeout } from "./timer-job-runner.js";
|
||||
export { maybeNotifyIsolatedAgentSetupTimeout } from "./timer-scheduler.js";
|
||||
export { applyJobResult } from "./timer-outcomes.js";
|
||||
export { applyTriggerRunResult } from "./timer-outcomes.js";
|
||||
export { applyTriggerRunResult } from "./timer-trigger.js";
|
||||
export { applyScriptRunResult } from "./timer-outcomes.js";
|
||||
export { applyTriggerNoFireResult } from "./timer-outcomes.js";
|
||||
export { armTimer } from "./timer-scheduler.js";
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Cron stagger tests cover deterministic schedule spreading across jobs.
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeCronStaggerMs, resolveCronStaggerMs } from "./stagger.js";
|
||||
|
||||
@@ -40,6 +41,7 @@ describe("cron stagger helpers", () => {
|
||||
expect(normalizeCronStaggerMs("abc")).toBeUndefined();
|
||||
expect(normalizeCronStaggerMs("1e3")).toBeUndefined();
|
||||
expect(normalizeCronStaggerMs("0x10")).toBeUndefined();
|
||||
expect(normalizeCronStaggerMs(MAX_DATE_TIMESTAMP_MS + 1)).toBeUndefined();
|
||||
expect(normalizeCronStaggerMs(Number.MAX_SAFE_INTEGER + 1)).toBeUndefined();
|
||||
});
|
||||
|
||||
|
||||
+5
-1
@@ -1,4 +1,8 @@
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import {
|
||||
asSafeIntegerInRange,
|
||||
MAX_DATE_TIMESTAMP_MS,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
/** Resolves deterministic cron stagger windows for recurring schedules. */
|
||||
import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js";
|
||||
import type { CronSchedule } from "./types.js";
|
||||
@@ -53,7 +57,7 @@ export function normalizeCronStaggerMs(raw: unknown): number | undefined {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = Math.max(0, Math.floor(numeric));
|
||||
return Number.isSafeInteger(normalized) ? normalized : undefined;
|
||||
return asSafeIntegerInRange(normalized, { max: MAX_DATE_TIMESTAMP_MS });
|
||||
}
|
||||
|
||||
/** Returns the default anti-thundering-herd stagger for top-of-hour recurring schedules. */
|
||||
|
||||
+35
-36
@@ -110,28 +110,27 @@ function mergeFailureDestinationProjection(
|
||||
}
|
||||
// Empty SQLite sentinels preserve explicit undefined fields for failure
|
||||
// destination overrides; project them back into the config sidecar shape.
|
||||
const delivery: Record<string, unknown> =
|
||||
isRecord(configJob.delivery) && !Array.isArray(configJob.delivery)
|
||||
? { ...configJob.delivery }
|
||||
: projectedJob?.delivery
|
||||
? {
|
||||
mode: projectedJob.delivery.mode,
|
||||
...(projectedJob.delivery.channel ? { channel: projectedJob.delivery.channel } : {}),
|
||||
...(projectedJob.delivery.to ? { to: projectedJob.delivery.to } : {}),
|
||||
...(projectedJob.delivery.threadId !== undefined
|
||||
? { threadId: projectedJob.delivery.threadId }
|
||||
: {}),
|
||||
...(projectedJob.delivery.accountId
|
||||
? { accountId: projectedJob.delivery.accountId }
|
||||
: {}),
|
||||
...(projectedJob.delivery.bestEffort !== undefined
|
||||
? { bestEffort: projectedJob.delivery.bestEffort }
|
||||
: {}),
|
||||
...(projectedJob.delivery.completionDestination
|
||||
? { completionDestination: projectedJob.delivery.completionDestination }
|
||||
: {}),
|
||||
}
|
||||
: {};
|
||||
const delivery: Record<string, unknown> = isRecord(configJob.delivery)
|
||||
? { ...configJob.delivery }
|
||||
: projectedJob?.delivery
|
||||
? {
|
||||
mode: projectedJob.delivery.mode,
|
||||
...(projectedJob.delivery.channel ? { channel: projectedJob.delivery.channel } : {}),
|
||||
...(projectedJob.delivery.to ? { to: projectedJob.delivery.to } : {}),
|
||||
...(projectedJob.delivery.threadId !== undefined
|
||||
? { threadId: projectedJob.delivery.threadId }
|
||||
: {}),
|
||||
...(projectedJob.delivery.accountId
|
||||
? { accountId: projectedJob.delivery.accountId }
|
||||
: {}),
|
||||
...(projectedJob.delivery.bestEffort !== undefined
|
||||
? { bestEffort: projectedJob.delivery.bestEffort }
|
||||
: {}),
|
||||
...(projectedJob.delivery.completionDestination
|
||||
? { completionDestination: projectedJob.delivery.completionDestination }
|
||||
: {}),
|
||||
}
|
||||
: {};
|
||||
const nextFailureDestination = isRecord(delivery.failureDestination)
|
||||
? { ...delivery.failureDestination }
|
||||
: {};
|
||||
@@ -227,7 +226,7 @@ export function assertCronStoreCanPersist(store: CronStoreFile): void {
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleFromRow(row: CronJobRow): CronSchedule | null {
|
||||
function scheduleFromRow(row: CronJobRow, jobJson: Record<string, unknown>): CronSchedule | null {
|
||||
if (row.schedule_kind === "at" && row.at) {
|
||||
return { kind: "at", at: row.at };
|
||||
}
|
||||
@@ -254,7 +253,7 @@ function scheduleFromRow(row: CronJobRow): CronSchedule | null {
|
||||
};
|
||||
}
|
||||
if (row.schedule_kind === "stream") {
|
||||
const schedule = asOptionalObjectRecord(safeParseJson(row.job_json))?.schedule;
|
||||
const schedule = jobJson.schedule;
|
||||
if (!isRecord(schedule) || schedule.kind !== "stream" || !Array.isArray(schedule.command)) {
|
||||
return null;
|
||||
}
|
||||
@@ -263,9 +262,9 @@ function scheduleFromRow(row: CronJobRow): CronSchedule | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function pacingFromRow(row: CronJobRow): CronPacing | undefined {
|
||||
const pacing = asOptionalObjectRecord(safeParseJson(row.job_json))?.pacing;
|
||||
if (!isRecord(pacing) || Array.isArray(pacing)) {
|
||||
function pacingFromJobJson(jobJson: Record<string, unknown>): CronPacing | undefined {
|
||||
const pacing = jobJson.pacing;
|
||||
if (!isRecord(pacing)) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
@@ -274,18 +273,17 @@ function pacingFromRow(row: CronJobRow): CronPacing | undefined {
|
||||
};
|
||||
}
|
||||
|
||||
function rowToCronJob(row: CronJobRow): CronStoredJob | null {
|
||||
const jobJson = asOptionalObjectRecord(safeParseJson(row.job_json)) ?? {};
|
||||
function rowToCronJob(row: CronJobRow, jobJson: Record<string, unknown>): CronStoredJob | null {
|
||||
const jsonOwner = isRecord(jobJson.owner) ? jobJson.owner : undefined;
|
||||
const ownerAccountId = normalizeOptionalAccountId(
|
||||
typeof jsonOwner?.accountId === "string" ? jsonOwner.accountId : undefined,
|
||||
);
|
||||
const schedule = scheduleFromRow(row);
|
||||
const schedule = scheduleFromRow(row, jobJson);
|
||||
const payload = payloadFromRow(row);
|
||||
const delivery = deliveryFromRow(row);
|
||||
const failureAlert = failureAlertFromRow(row);
|
||||
const trigger = triggerFromRow(row);
|
||||
const pacing = pacingFromRow(row);
|
||||
const pacing = pacingFromJobJson(jobJson);
|
||||
const scheduledToolPolicy = normalizeCronScheduledToolPolicy(jobJson.scheduledToolPolicy);
|
||||
const toolsAllowProvenance =
|
||||
isRecord(jobJson.toolsAllowProvenance) &&
|
||||
@@ -342,7 +340,7 @@ export function projectCronJobThroughStorageCodec(job: CronStoredJob): CronStore
|
||||
throw new Error(`cannot project invalid cron job ${job.id}`);
|
||||
}
|
||||
const row = bindCronJobRow("config-revision", normalized, 0) as CronJobRow;
|
||||
const projected = rowToCronJob(row);
|
||||
const projected = rowToCronJob(row, asOptionalObjectRecord(safeParseJson(row.job_json)) ?? {});
|
||||
if (!projected) {
|
||||
throw new Error(`cannot project cron job ${job.id} through storage codecs`);
|
||||
}
|
||||
@@ -487,10 +485,11 @@ export function loadedCronStoreFromRows(rows: CronJobRow[]): LoadedCronStore {
|
||||
const invalidConfigRows: LoadedCronStore["invalidConfigRows"] = [];
|
||||
|
||||
for (const [index, row] of rows.entries()) {
|
||||
const job = rowToCronJob(row);
|
||||
const parsedJobJson = asOptionalObjectRecord(safeParseJson(row.job_json));
|
||||
const jobJson = parsedJobJson ?? {};
|
||||
const job = rowToCronJob(row, jobJson);
|
||||
const configJob = mergeFailureDestinationProjection(
|
||||
asOptionalObjectRecord(safeParseJson(row.job_json)) ??
|
||||
(job ? stripJobRuntimeFields(job) : {}),
|
||||
parsedJobJson ?? (job ? stripJobRuntimeFields(job) : {}),
|
||||
job,
|
||||
);
|
||||
const runtimeEntry = {
|
||||
@@ -504,7 +503,7 @@ export function loadedCronStoreFromRows(rows: CronJobRow[]): LoadedCronStore {
|
||||
sourceIndex: index,
|
||||
reason:
|
||||
getInvalidPersistedCronJobReason(configJob) ??
|
||||
(scheduleFromRow(row) ? "invalid-payload" : "invalid-schedule"),
|
||||
(scheduleFromRow(row, jobJson) ? "invalid-payload" : "invalid-schedule"),
|
||||
job: configJob,
|
||||
...(runtimeEntry.state ? { state: runtimeEntry.state } : {}),
|
||||
...(runtimeEntry.updatedAtMs !== undefined
|
||||
|
||||
+43
-38
@@ -1,6 +1,10 @@
|
||||
/** Read-side cron codec between task-ledger detail and the stable run-history wire shape.
|
||||
* Deliberately free of agent/runtime imports so history reads stay dependency-light;
|
||||
* the event->entry write codec lives in task-run-event-codec.ts. */
|
||||
import {
|
||||
asSafeIntegerInRange,
|
||||
MAX_DATE_TIMESTAMP_MS,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import {
|
||||
FAILOVER_REASONS,
|
||||
@@ -24,14 +28,38 @@ function toJsonValue(value: unknown): JsonValue | undefined {
|
||||
return serialized === undefined ? undefined : (JSON.parse(serialized) as JsonValue);
|
||||
}
|
||||
|
||||
function isJsonObject(value: JsonValue | undefined): value is { [key: string]: JsonValue } {
|
||||
function isJsonObject(value: unknown): value is { [key: string]: JsonValue } {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isCronRunStatus(value: unknown): value is CronRunStatus {
|
||||
function normalizeTimestamp(value: unknown): number | undefined {
|
||||
return asSafeIntegerInRange(value, { min: 0, max: MAX_DATE_TIMESTAMP_MS });
|
||||
}
|
||||
|
||||
export function isCronRunStatus(value: unknown): value is CronRunStatus {
|
||||
return value === "ok" || value === "error" || value === "skipped";
|
||||
}
|
||||
|
||||
export function isCronDeliveryStatus(value: unknown): value is CronDeliveryStatus {
|
||||
return ["delivered", "not-delivered", "unknown", "not-requested"].includes(
|
||||
value as CronDeliveryStatus,
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeUsage(value: unknown): CronRunLogEntry["usage"] {
|
||||
if (!isJsonObject(value)) {
|
||||
return undefined;
|
||||
}
|
||||
const usage = {
|
||||
input_tokens: asSafeIntegerInRange(value.input_tokens, { min: 0 }),
|
||||
output_tokens: asSafeIntegerInRange(value.output_tokens, { min: 0 }),
|
||||
total_tokens: asSafeIntegerInRange(value.total_tokens, { min: 0 }),
|
||||
cache_read_tokens: asSafeIntegerInRange(value.cache_read_tokens, { min: 0 }),
|
||||
cache_write_tokens: asSafeIntegerInRange(value.cache_write_tokens, { min: 0 }),
|
||||
};
|
||||
return Object.values(usage).some((tokenCount) => tokenCount !== undefined) ? usage : undefined;
|
||||
}
|
||||
|
||||
function normalizeCronRunLogErrorReason(value: unknown): FailoverReason | undefined {
|
||||
return typeof value === "string" && CRON_FAILOVER_REASONS.has(value as FailoverReason)
|
||||
? (value as FailoverReason)
|
||||
@@ -54,17 +82,14 @@ export function parseCronRunLogEntryObject(
|
||||
if (typeof entryObj.jobId !== "string" || entryObj.jobId.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (typeof entryObj.ts !== "number" || !Number.isFinite(entryObj.ts)) {
|
||||
const ts = normalizeTimestamp(entryObj.ts);
|
||||
if (ts === undefined) {
|
||||
return null;
|
||||
}
|
||||
if (jobId && entryObj.jobId !== jobId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const usage =
|
||||
entryObj.usage && typeof entryObj.usage === "object"
|
||||
? (entryObj.usage as Record<string, unknown>)
|
||||
: undefined;
|
||||
const normalizedError = typeof entryObj.error === "string" ? entryObj.error : undefined;
|
||||
const normalizedProvider =
|
||||
typeof entryObj.provider === "string" && entryObj.provider.trim()
|
||||
@@ -72,43 +97,28 @@ export function parseCronRunLogEntryObject(
|
||||
: undefined;
|
||||
// Diagnostics are redacted at authoring; this read/migration path only normalizes stored shape.
|
||||
const entry: CronRunLogEntry = {
|
||||
ts: entryObj.ts,
|
||||
ts,
|
||||
jobId: entryObj.jobId,
|
||||
action: "finished",
|
||||
status: entryObj.status,
|
||||
status: isCronRunStatus(entryObj.status) ? entryObj.status : undefined,
|
||||
error: normalizedError,
|
||||
errorReason: normalizeCronRunLogErrorReason(entryObj.errorReason) ?? undefined,
|
||||
summary: entryObj.summary,
|
||||
summary: typeof entryObj.summary === "string" ? entryObj.summary : undefined,
|
||||
runId: typeof entryObj.runId === "string" && entryObj.runId.trim() ? entryObj.runId : undefined,
|
||||
diagnostics: normalizeCronRunDiagnostics(entryObj.diagnostics),
|
||||
runAtMs: entryObj.runAtMs,
|
||||
durationMs: entryObj.durationMs,
|
||||
nextRunAtMs: entryObj.nextRunAtMs,
|
||||
runAtMs: normalizeTimestamp(entryObj.runAtMs),
|
||||
durationMs: asSafeIntegerInRange(entryObj.durationMs, { min: 0 }),
|
||||
nextRunAtMs: normalizeTimestamp(entryObj.nextRunAtMs),
|
||||
triggerFired: entryObj.triggerFired === true ? true : undefined,
|
||||
model: typeof entryObj.model === "string" && entryObj.model.trim() ? entryObj.model : undefined,
|
||||
provider: normalizedProvider,
|
||||
usage: usage
|
||||
? {
|
||||
input_tokens: typeof usage.input_tokens === "number" ? usage.input_tokens : undefined,
|
||||
output_tokens: typeof usage.output_tokens === "number" ? usage.output_tokens : undefined,
|
||||
total_tokens: typeof usage.total_tokens === "number" ? usage.total_tokens : undefined,
|
||||
cache_read_tokens:
|
||||
typeof usage.cache_read_tokens === "number" ? usage.cache_read_tokens : undefined,
|
||||
cache_write_tokens:
|
||||
typeof usage.cache_write_tokens === "number" ? usage.cache_write_tokens : undefined,
|
||||
}
|
||||
: undefined,
|
||||
usage: normalizeUsage(entryObj.usage),
|
||||
};
|
||||
if (typeof entryObj.delivered === "boolean") {
|
||||
entry.delivered = entryObj.delivered;
|
||||
}
|
||||
if (
|
||||
entryObj.deliveryStatus === "delivered" ||
|
||||
entryObj.deliveryStatus === "not-delivered" ||
|
||||
entryObj.deliveryStatus === "unknown" ||
|
||||
entryObj.deliveryStatus === "not-requested"
|
||||
) {
|
||||
entry.deliveryStatus = entryObj.deliveryStatus as CronDeliveryStatus;
|
||||
if (isCronDeliveryStatus(entryObj.deliveryStatus)) {
|
||||
entry.deliveryStatus = entryObj.deliveryStatus;
|
||||
}
|
||||
if (typeof entryObj.deliveryError === "string") {
|
||||
entry.deliveryError = entryObj.deliveryError;
|
||||
@@ -122,12 +132,7 @@ export function parseCronRunLogEntryObject(
|
||||
status?: unknown;
|
||||
error?: unknown;
|
||||
};
|
||||
if (
|
||||
failureNotificationDelivery.status === "delivered" ||
|
||||
failureNotificationDelivery.status === "not-delivered" ||
|
||||
failureNotificationDelivery.status === "unknown" ||
|
||||
failureNotificationDelivery.status === "not-requested"
|
||||
) {
|
||||
if (isCronDeliveryStatus(failureNotificationDelivery.status)) {
|
||||
entry.failureNotificationDelivery = {
|
||||
status: failureNotificationDelivery.status,
|
||||
...(typeof failureNotificationDelivery.delivered === "boolean"
|
||||
@@ -139,7 +144,7 @@ export function parseCronRunLogEntryObject(
|
||||
};
|
||||
}
|
||||
}
|
||||
if (entryObj.delivery && typeof entryObj.delivery === "object") {
|
||||
if (isJsonObject(entryObj.delivery)) {
|
||||
entry.delivery = entryObj.delivery;
|
||||
}
|
||||
if (typeof entryObj.sessionId === "string" && entryObj.sessionId.trim()) {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { FAILOVER_REASONS } from "../../packages/gateway-protocol/src/failover-reasons.js";
|
||||
import { saveTaskRegistryStateToSqlite } from "../tasks/task-registry.store.sqlite.js";
|
||||
@@ -441,16 +442,27 @@ describe("cron task run history", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the internal store key out of the legacy wire record", () => {
|
||||
it("allowlists the legacy wire record", () => {
|
||||
const storeKey = "/internal/cron/store";
|
||||
const task = taskFromEntry(
|
||||
{ ts: 100, jobId: JOB_ID, action: "finished", status: "ok" },
|
||||
1,
|
||||
storeKey,
|
||||
);
|
||||
task.detail = {
|
||||
...(task.detail as Record<string, TaskRecord["detail"]>),
|
||||
internalFutureField: "secret",
|
||||
triggerState: { secret: true },
|
||||
delivery: "malformed",
|
||||
failureNotificationDelivery: { status: "invalid", internal: "secret" },
|
||||
};
|
||||
const entry = cronTaskRecordToRunLogEntry(task);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(Object.hasOwn(entry ?? {}, "storeKey")).toBe(false);
|
||||
expect(Object.hasOwn(entry ?? {}, "internalFutureField")).toBe(false);
|
||||
expect(Object.hasOwn(entry ?? {}, "triggerState")).toBe(false);
|
||||
expect(entry?.delivery).toBeUndefined();
|
||||
expect(entry?.failureNotificationDelivery).toBeUndefined();
|
||||
});
|
||||
|
||||
it("locks the serialized detail shape: kind first, status second", () => {
|
||||
@@ -494,6 +506,47 @@ describe("cron task run history", () => {
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects invalid legacy run-history scalar and timestamp fields", () => {
|
||||
const base = { ts: 100, jobId: JOB_ID, action: "finished" } as const;
|
||||
expect(
|
||||
parseCronRunLogEntryObject({
|
||||
...base,
|
||||
status: "invalid",
|
||||
summary: 42,
|
||||
runAtMs: -1,
|
||||
durationMs: 1.5,
|
||||
nextRunAtMs: MAX_DATE_TIMESTAMP_MS + 1,
|
||||
delivery: [],
|
||||
usage: { input_tokens: Number.NaN, output_tokens: -1 },
|
||||
}),
|
||||
).toEqual({
|
||||
...base,
|
||||
status: undefined,
|
||||
error: undefined,
|
||||
errorReason: undefined,
|
||||
summary: undefined,
|
||||
runId: undefined,
|
||||
diagnostics: undefined,
|
||||
runAtMs: undefined,
|
||||
durationMs: undefined,
|
||||
nextRunAtMs: undefined,
|
||||
triggerFired: undefined,
|
||||
model: undefined,
|
||||
provider: undefined,
|
||||
usage: undefined,
|
||||
});
|
||||
expect(parseCronRunLogEntryObject({ ...base, usage: [] })?.usage).toBeUndefined();
|
||||
expect(parseCronRunLogEntryObject({ ...base, usage: { input_tokens: 0 } })?.usage).toEqual({
|
||||
input_tokens: 0,
|
||||
output_tokens: undefined,
|
||||
total_tokens: undefined,
|
||||
cache_read_tokens: undefined,
|
||||
cache_write_tokens: undefined,
|
||||
});
|
||||
expect(parseCronRunLogEntryObject({ ...base, ts: MAX_DATE_TIMESTAMP_MS })).not.toBeNull();
|
||||
expect(parseCronRunLogEntryObject({ ...base, ts: MAX_DATE_TIMESTAMP_MS + 1 })).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves every canonical failover reason in stored run history", () => {
|
||||
for (const errorReason of FAILOVER_REASONS) {
|
||||
const entry = {
|
||||
|
||||
@@ -7,7 +7,12 @@ import { uniqueValues } from "@openclaw/normalization-core/string-normalization"
|
||||
import { listTaskRegistryRecordsByRuntimeSourceIdFromSqlite } from "../tasks/task-registry.store.sqlite.js";
|
||||
import type { TaskRecord } from "../tasks/task-registry.types.js";
|
||||
import type { CronRunLogEntry } from "./run-log-types.js";
|
||||
import { cronTaskRecordStoreKey, cronTaskRecordToRunLogEntry } from "./task-run-detail.js";
|
||||
import {
|
||||
cronTaskRecordStoreKey,
|
||||
cronTaskRecordToRunLogEntry,
|
||||
isCronDeliveryStatus,
|
||||
isCronRunStatus,
|
||||
} from "./task-run-detail.js";
|
||||
import type { CronDeliveryStatus, CronRunStatus } from "./types.js";
|
||||
|
||||
type CronRunHistorySortDir = "asc" | "desc";
|
||||
@@ -63,19 +68,6 @@ function normalizeStatuses(options: ReadCronTaskRunHistoryPageOptions): CronRunS
|
||||
return isCronRunStatus(options.status) ? [options.status] : null;
|
||||
}
|
||||
|
||||
function isCronRunStatus(value: unknown): value is CronRunStatus {
|
||||
return value === "ok" || value === "error" || value === "skipped";
|
||||
}
|
||||
|
||||
function isCronDeliveryStatus(value: unknown): value is CronDeliveryStatus {
|
||||
return (
|
||||
value === "delivered" ||
|
||||
value === "not-delivered" ||
|
||||
value === "unknown" ||
|
||||
value === "not-requested"
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeDeliveryStatuses(
|
||||
options: ReadCronTaskRunHistoryPageOptions,
|
||||
): CronDeliveryStatus[] | null {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { toErrorObject } from "@openclaw/normalization-core/error-coercion";
|
||||
import { errorBackoffMs } from "../cron/service/jobs.js";
|
||||
import { errorBackoffMs } from "../cron/service/jobs-scheduling.js";
|
||||
import { cronStreamScheduleKey } from "../cron/stream-schedule.js";
|
||||
import type { CronJob, CronJobState } from "../cron/types.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
|
||||
@@ -40,11 +40,6 @@ type CronAgentResolver = (requested?: string | null) => {
|
||||
cfg: OpenClawConfig;
|
||||
};
|
||||
|
||||
type CronWebhookTarget = {
|
||||
url: string;
|
||||
source: "completionDestination";
|
||||
};
|
||||
|
||||
type CronFailureAlertParams = {
|
||||
deps: CliDeps;
|
||||
logger: CronLogger;
|
||||
@@ -130,27 +125,20 @@ function redactCommandCronEventForExternalDelivery(evt: CronEvent, job?: CronJob
|
||||
return redacted;
|
||||
}
|
||||
|
||||
/** Resolves detached completion-destination webhooks. */
|
||||
function resolveCronWebhookTargets(params: {
|
||||
function resolveCronCompletionWebhook(params: {
|
||||
delivery?: {
|
||||
mode?: string;
|
||||
to?: string;
|
||||
completionDestination?: { mode?: string; to?: string };
|
||||
};
|
||||
}): CronWebhookTarget[] {
|
||||
const targets: CronWebhookTarget[] = [];
|
||||
const mode = normalizeOptionalLowercaseString(params.delivery?.mode);
|
||||
const completionMode = normalizeOptionalLowercaseString(
|
||||
params.delivery?.completionDestination?.mode,
|
||||
);
|
||||
if (mode === "announce" && completionMode === "webhook") {
|
||||
const url = normalizeHttpWebhookUrl(params.delivery?.completionDestination?.to);
|
||||
if (url) {
|
||||
targets.push({ url, source: "completionDestination" });
|
||||
}
|
||||
}): string | undefined {
|
||||
if (
|
||||
normalizeOptionalLowercaseString(params.delivery?.mode) !== "announce" ||
|
||||
normalizeOptionalLowercaseString(params.delivery?.completionDestination?.mode) !== "webhook"
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return targets;
|
||||
return normalizeHttpWebhookUrl(params.delivery?.completionDestination?.to) ?? undefined;
|
||||
}
|
||||
|
||||
function buildCronWebhookHeaders(webhookToken?: string): Record<string, string> {
|
||||
@@ -460,7 +448,7 @@ export function dispatchGatewayCronFinishedNotifications(params: {
|
||||
params.job?.payload.kind === "script"
|
||||
? normalizeOptionalString(redactedWebhookEvent.summary)
|
||||
: params.evt.summary;
|
||||
const webhookTargets = resolveCronWebhookTargets({
|
||||
const completionWebhookUrl = resolveCronCompletionWebhook({
|
||||
delivery:
|
||||
params.job?.delivery && typeof params.job.delivery.mode === "string"
|
||||
? {
|
||||
@@ -486,27 +474,23 @@ export function dispatchGatewayCronFinishedNotifications(params: {
|
||||
|
||||
// Script notify is carried as the completion summary, so its absence uses
|
||||
// the same silent-summary suppression path as NO_REPLY output.
|
||||
if (completionSummary || params.evt.status === "error") {
|
||||
for (const webhookTarget of webhookTargets) {
|
||||
const payload = buildCronFinishedWebhookPayload(redactedWebhookEvent);
|
||||
// Completion notification fanout is best-effort; the cron service has
|
||||
// already recorded the run result and must not wait on slow webhooks.
|
||||
dispatchDetachedCronNotification({
|
||||
jobId: params.evt.jobId,
|
||||
logger: params.logger,
|
||||
deliver: () =>
|
||||
postCronWebhook({
|
||||
webhookUrl: webhookTarget.url,
|
||||
webhookToken,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
payload,
|
||||
logContext: { jobId: params.evt.jobId, source: webhookTarget.source },
|
||||
blockedLog: "cron: webhook delivery blocked by SSRF guard",
|
||||
failedLog: "cron: webhook delivery failed",
|
||||
logger: params.logger,
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (completionWebhookUrl && (completionSummary || params.evt.status === "error")) {
|
||||
const payload = buildCronFinishedWebhookPayload(redactedWebhookEvent);
|
||||
dispatchDetachedCronNotification({
|
||||
jobId: params.evt.jobId,
|
||||
logger: params.logger,
|
||||
deliver: () =>
|
||||
postCronWebhook({
|
||||
webhookUrl: completionWebhookUrl,
|
||||
webhookToken,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
payload,
|
||||
logContext: { jobId: params.evt.jobId, source: "completionDestination" },
|
||||
blockedLog: "cron: webhook delivery blocked by SSRF guard",
|
||||
failedLog: "cron: webhook delivery failed",
|
||||
logger: params.logger,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
dispatchCronFailureDestinationNotifications({
|
||||
@@ -573,41 +557,31 @@ function dispatchCronFailureDestinationNotifications(params: {
|
||||
return;
|
||||
}
|
||||
|
||||
if (failureDest.mode === "announce") {
|
||||
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(job.agentId);
|
||||
dispatchDetachedCronNotification({
|
||||
jobId: job.id,
|
||||
logger: params.logger,
|
||||
deliver: () =>
|
||||
sendFailureNotificationAnnounce(
|
||||
params.deps,
|
||||
runtimeConfig,
|
||||
agentId,
|
||||
job.id,
|
||||
{
|
||||
channel: failureDest.channel,
|
||||
to: failureDest.to,
|
||||
accountId: failureDest.accountId,
|
||||
sessionKey: deliverySessionKey,
|
||||
// A configured failure route is already explicit; keep the cron run
|
||||
// session only for context, not for reattaching the primary topic.
|
||||
inheritSessionThread: false,
|
||||
},
|
||||
{
|
||||
text: appendCronRunStarted(
|
||||
`⚠️ ${failurePayload.message}`,
|
||||
params.evt.runAtMs,
|
||||
runtimeConfig,
|
||||
),
|
||||
},
|
||||
),
|
||||
});
|
||||
if (failureDest.mode !== "announce") {
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const primaryPlan = resolveCronDeliveryPlan(job);
|
||||
if (primaryPlan.mode !== "announce" || !primaryPlan.requested) {
|
||||
const announceTarget = failureDest
|
||||
? {
|
||||
channel: failureDest.channel,
|
||||
to: failureDest.to,
|
||||
accountId: failureDest.accountId,
|
||||
sessionKey: deliverySessionKey,
|
||||
// Explicit failure routes keep run context without inheriting the primary topic.
|
||||
inheritSessionThread: false,
|
||||
}
|
||||
: primaryPlan.mode === "announce" && primaryPlan.requested
|
||||
? {
|
||||
channel: primaryPlan.channel,
|
||||
to: primaryPlan.to,
|
||||
accountId: primaryPlan.accountId,
|
||||
threadId: primaryPlan.threadId,
|
||||
sessionKey: deliverySessionKey,
|
||||
}
|
||||
: undefined;
|
||||
if (!announceTarget) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -616,25 +590,12 @@ function dispatchCronFailureDestinationNotifications(params: {
|
||||
jobId: job.id,
|
||||
logger: params.logger,
|
||||
deliver: () =>
|
||||
sendFailureNotificationAnnounce(
|
||||
params.deps,
|
||||
runtimeConfig,
|
||||
agentId,
|
||||
job.id,
|
||||
{
|
||||
channel: primaryPlan.channel,
|
||||
to: primaryPlan.to,
|
||||
accountId: primaryPlan.accountId,
|
||||
threadId: primaryPlan.threadId,
|
||||
sessionKey: deliverySessionKey,
|
||||
},
|
||||
{
|
||||
text: appendCronRunStarted(
|
||||
`⚠️ ${failurePayload.message}`,
|
||||
params.evt.runAtMs,
|
||||
runtimeConfig,
|
||||
),
|
||||
},
|
||||
),
|
||||
sendFailureNotificationAnnounce(params.deps, runtimeConfig, agentId, job.id, announceTarget, {
|
||||
text: appendCronRunStarted(
|
||||
`⚠️ ${failurePayload.message}`,
|
||||
params.evt.runAtMs,
|
||||
runtimeConfig,
|
||||
),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
+119
-132
@@ -222,6 +222,86 @@ function sanitizeCronHeartbeatOverride(
|
||||
return heartbeat?.target === "last" ? omitExplicitHeartbeatDestination(heartbeat) : heartbeat;
|
||||
}
|
||||
|
||||
async function finalizeCronCompletionAnnouncement(params: {
|
||||
job: CronJob;
|
||||
text?: string;
|
||||
abortSignal?: AbortSignal;
|
||||
deps: CliDeps;
|
||||
resolveCronAgent: (requested?: string | null) => { agentId: string; cfg: OpenClawConfig };
|
||||
logger: ReturnType<typeof getChildLogger>;
|
||||
label: string;
|
||||
traceResolvedFailure?: boolean;
|
||||
}) {
|
||||
const plan = resolveCronDeliveryPlan(params.job);
|
||||
const delivery = {
|
||||
intended: pickDefined(
|
||||
{
|
||||
channel: plan.channel,
|
||||
to: plan.to,
|
||||
accountId: plan.accountId,
|
||||
threadId: plan.threadId,
|
||||
source: "explicit" as const,
|
||||
},
|
||||
["channel", "to", "accountId", "threadId", "source"],
|
||||
),
|
||||
};
|
||||
if (plan.mode !== "announce" || params.text === undefined) {
|
||||
return { deliveryAttempted: false, delivered: false, delivery };
|
||||
}
|
||||
|
||||
const { agentId, cfg } = params.resolveCronAgent(params.job.agentId);
|
||||
try {
|
||||
await sendCronAnnouncePayloadStrict({
|
||||
deps: params.deps,
|
||||
cfg,
|
||||
agentId,
|
||||
jobId: params.job.id,
|
||||
target: {
|
||||
channel: plan.channel,
|
||||
to: plan.to,
|
||||
threadId: plan.threadId,
|
||||
accountId: plan.accountId,
|
||||
sessionKey: resolveCronDeliverySessionKey(params.job),
|
||||
},
|
||||
payload: { text: params.text },
|
||||
abortSignal: params.abortSignal ?? new AbortController().signal,
|
||||
});
|
||||
return {
|
||||
deliveryAttempted: true,
|
||||
delivered: true,
|
||||
delivery: { ...delivery, delivered: true },
|
||||
};
|
||||
} catch (err) {
|
||||
const deliveryError = formatErrorMessage(err);
|
||||
params.logger.warn(
|
||||
{ jobId: params.job.id, err: deliveryError },
|
||||
`cron: ${params.label} delivery failed`,
|
||||
);
|
||||
return {
|
||||
deliveryAttempted: true,
|
||||
delivered: false,
|
||||
deliveryError,
|
||||
delivery: {
|
||||
...delivery,
|
||||
delivered: false,
|
||||
...(params.traceResolvedFailure
|
||||
? {
|
||||
resolved: {
|
||||
channel: plan.channel,
|
||||
to: plan.to,
|
||||
accountId: plan.accountId,
|
||||
threadId: plan.threadId,
|
||||
source: "explicit" as const,
|
||||
ok: false,
|
||||
error: deliveryError,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Map internal CronJob to the public plugin SDK shape. */
|
||||
function toPluginCronJob(job: CronJob): PluginHookGatewayCronJob {
|
||||
return {
|
||||
@@ -746,104 +826,45 @@ export function buildGatewayCronService(params: {
|
||||
abortSignal,
|
||||
nowMs: Date.now,
|
||||
});
|
||||
const plan = resolveCronDeliveryPlan(job);
|
||||
const deliveryTrace = {
|
||||
intended: pickDefined(
|
||||
{
|
||||
channel: plan.channel,
|
||||
to: plan.to,
|
||||
threadId: plan.threadId,
|
||||
accountId: plan.accountId,
|
||||
source: "explicit" as const,
|
||||
},
|
||||
["channel", "to", "accountId", "threadId", "source"],
|
||||
),
|
||||
};
|
||||
const summaryIsSilent =
|
||||
typeof result.summary === "string" && isSilentReplyText(result.summary, SILENT_REPLY_TOKEN);
|
||||
if (summaryIsSilent) {
|
||||
const { summary: _summary, ...silentResult } = result;
|
||||
return {
|
||||
...silentResult,
|
||||
deliveryAttempted: false,
|
||||
delivered: false,
|
||||
delivery: deliveryTrace,
|
||||
};
|
||||
}
|
||||
const shouldAnnounce =
|
||||
plan.mode === "announce" && typeof result.summary === "string" && result.summary.trim();
|
||||
if (!shouldAnnounce) {
|
||||
return {
|
||||
...result,
|
||||
deliveryAttempted: false,
|
||||
delivered: false,
|
||||
delivery: deliveryTrace,
|
||||
};
|
||||
}
|
||||
const message = isCommandCronJob(job)
|
||||
? redactCronCommandSummaryForExternalDelivery(result.summary)
|
||||
: result.summary;
|
||||
if (typeof message !== "string") {
|
||||
return {
|
||||
...result,
|
||||
deliveryAttempted: false,
|
||||
delivered: false,
|
||||
delivery: deliveryTrace,
|
||||
};
|
||||
}
|
||||
const { agentId, cfg: runtimeConfig } = resolveCronAgent(job.agentId);
|
||||
try {
|
||||
await sendCronAnnouncePayloadStrict({
|
||||
const completion = await finalizeCronCompletionAnnouncement({
|
||||
job,
|
||||
deps: params.deps,
|
||||
cfg: runtimeConfig,
|
||||
agentId,
|
||||
jobId: job.id,
|
||||
target: {
|
||||
channel: plan.channel,
|
||||
to: plan.to,
|
||||
threadId: plan.threadId,
|
||||
accountId: plan.accountId,
|
||||
sessionKey: resolveCronDeliverySessionKey(job),
|
||||
},
|
||||
payload: { text: message },
|
||||
abortSignal: abortSignal ?? new AbortController().signal,
|
||||
resolveCronAgent,
|
||||
logger: cronLogger,
|
||||
label: "command",
|
||||
});
|
||||
return {
|
||||
...result,
|
||||
deliveryAttempted: true,
|
||||
delivered: true,
|
||||
delivery: {
|
||||
...deliveryTrace,
|
||||
delivered: true,
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
const error = formatErrorMessage(err);
|
||||
return { ...silentResult, ...completion };
|
||||
}
|
||||
const completion = await finalizeCronCompletionAnnouncement({
|
||||
job,
|
||||
text:
|
||||
typeof result.summary === "string" && result.summary.trim()
|
||||
? redactCronCommandSummaryForExternalDelivery(result.summary)
|
||||
: undefined,
|
||||
abortSignal,
|
||||
deps: params.deps,
|
||||
resolveCronAgent,
|
||||
logger: cronLogger,
|
||||
label: "command",
|
||||
traceResolvedFailure: true,
|
||||
});
|
||||
if ("deliveryError" in completion) {
|
||||
const { deliveryError, ...deliveryResult } = completion;
|
||||
const requiredDeliveryFailed = job.delivery?.bestEffort === false && result.status === "ok";
|
||||
cronLogger.warn({ jobId: job.id, err: error }, "cron: command delivery failed");
|
||||
return {
|
||||
...result,
|
||||
// Default announce delivery is best-effort, but an explicit
|
||||
// bestEffort:false keeps delivery inside the job's success contract.
|
||||
status: requiredDeliveryFailed ? ("error" as const) : result.status,
|
||||
...(requiredDeliveryFailed ? { error } : { deliveryError: error }),
|
||||
deliveryAttempted: true,
|
||||
delivered: false,
|
||||
delivery: {
|
||||
...deliveryTrace,
|
||||
delivered: false,
|
||||
resolved: {
|
||||
channel: plan.channel,
|
||||
to: plan.to,
|
||||
accountId: plan.accountId,
|
||||
threadId: plan.threadId,
|
||||
source: "explicit" as const,
|
||||
ok: false,
|
||||
error,
|
||||
},
|
||||
},
|
||||
...(requiredDeliveryFailed ? { error: deliveryError } : { deliveryError }),
|
||||
...deliveryResult,
|
||||
};
|
||||
}
|
||||
return { ...result, ...completion };
|
||||
},
|
||||
sendCronWebhook: async ({ job, event, abortSignal, deadlineAtMs, onDeliveryAccepted }) => {
|
||||
await sendGatewayCronWebhook({
|
||||
@@ -891,19 +912,6 @@ export function buildGatewayCronService(params: {
|
||||
}
|
||||
|
||||
const notify = execution.notify?.trim() ? execution.notify : undefined;
|
||||
const plan = resolveCronDeliveryPlan(job);
|
||||
const deliveryTrace = {
|
||||
intended: pickDefined(
|
||||
{
|
||||
channel: plan.channel,
|
||||
to: plan.to,
|
||||
accountId: plan.accountId,
|
||||
threadId: plan.threadId,
|
||||
source: "explicit" as const,
|
||||
},
|
||||
["channel", "to", "accountId", "threadId", "source"],
|
||||
),
|
||||
};
|
||||
const base = {
|
||||
status: "ok" as const,
|
||||
notify,
|
||||
@@ -911,47 +919,26 @@ export function buildGatewayCronService(params: {
|
||||
stateChanged: execution.stateChanged,
|
||||
...(execution.stateChanged ? { state: execution.state } : {}),
|
||||
nextCheck: execution.nextCheck,
|
||||
delivery: deliveryTrace,
|
||||
};
|
||||
if (job.sessionTarget === "main" || plan.mode !== "announce" || !notify) {
|
||||
return { ...base, deliveryAttempted: false, delivered: false };
|
||||
}
|
||||
|
||||
const { agentId, cfg: runtimeConfig } = resolveCronAgent(job.agentId);
|
||||
try {
|
||||
await sendCronAnnouncePayloadStrict({
|
||||
deps: params.deps,
|
||||
cfg: runtimeConfig,
|
||||
agentId,
|
||||
jobId: job.id,
|
||||
target: {
|
||||
channel: plan.channel,
|
||||
to: plan.to,
|
||||
threadId: plan.threadId,
|
||||
accountId: plan.accountId,
|
||||
sessionKey: resolveCronDeliverySessionKey(job),
|
||||
},
|
||||
payload: { text: notify },
|
||||
abortSignal: abortSignal ?? new AbortController().signal,
|
||||
});
|
||||
return {
|
||||
...base,
|
||||
deliveryAttempted: true,
|
||||
delivered: true,
|
||||
delivery: { ...deliveryTrace, delivered: true },
|
||||
};
|
||||
} catch (err) {
|
||||
const error = formatErrorMessage(err);
|
||||
cronLogger.warn({ jobId: job.id, err: error }, "cron: script payload delivery failed");
|
||||
const completion = await finalizeCronCompletionAnnouncement({
|
||||
job,
|
||||
text: job.sessionTarget === "main" ? undefined : notify,
|
||||
abortSignal,
|
||||
deps: params.deps,
|
||||
resolveCronAgent,
|
||||
logger: cronLogger,
|
||||
label: "script payload",
|
||||
});
|
||||
if ("deliveryError" in completion) {
|
||||
const { deliveryError, ...deliveryResult } = completion;
|
||||
return {
|
||||
...base,
|
||||
status: job.delivery?.bestEffort ? ("ok" as const) : ("error" as const),
|
||||
...(job.delivery?.bestEffort ? { deliveryError: error } : { error }),
|
||||
deliveryAttempted: true,
|
||||
delivered: false,
|
||||
delivery: { ...deliveryTrace, delivered: false },
|
||||
...(job.delivery?.bestEffort ? { deliveryError } : { error: deliveryError }),
|
||||
...deliveryResult,
|
||||
};
|
||||
}
|
||||
return { ...base, ...completion };
|
||||
},
|
||||
cleanupTimedOutAgentRun: async ({ job, execution }) => {
|
||||
if (!execution?.sessionId) {
|
||||
|
||||
@@ -8,6 +8,8 @@ describe("hasValidIsoCalendarComponents", () => {
|
||||
"2028-02-29T12:30:45.123456+01:30",
|
||||
"2028-02-29T24:00:00Z",
|
||||
"2028-02-29T24:00:00.0000Z",
|
||||
"+275760-09-13T00:00:00.000Z",
|
||||
"-271821-04-20T00:00:00.000Z",
|
||||
])("accepts valid calendar components in %s", (value) => {
|
||||
expect(hasValidIsoCalendarComponents(value)).toBe(true);
|
||||
});
|
||||
@@ -21,6 +23,8 @@ describe("hasValidIsoCalendarComponents", () => {
|
||||
"2026-07-05T12:60:00Z",
|
||||
"2026-07-05T12:00:60Z",
|
||||
"2026-7-05",
|
||||
"+275760-09-13T00:00:00.001Z",
|
||||
"275760-09-13T00:00:00.000Z",
|
||||
])("rejects invalid calendar components or shape in %s", (value) => {
|
||||
expect(hasValidIsoCalendarComponents(value)).toBe(false);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const ISO_ABSOLUTE_RE =
|
||||
/^(\d{4})-(\d{2})-(\d{2})(?:[Tt](\d{2}):(\d{2})(?::(\d{2})(\.\d+)?)?(?:[Zz]|[+-]\d{2}:?\d{2})?)?$/;
|
||||
/^([+-]\d{6}|\d{4})-(\d{2})-(\d{2})(?:[Tt](\d{2}):(\d{2})(?::(\d{2})(\.\d+)?)?(?:[Zz]|[+-]\d{2}:?\d{2})?)?$/;
|
||||
|
||||
/** Checks the calendar components of the ISO-like forms accepted by existing callers. */
|
||||
export function hasValidIsoCalendarComponents(raw: string): boolean {
|
||||
|
||||
+49
-1
@@ -261,7 +261,7 @@
|
||||
"tools": [
|
||||
{
|
||||
"deferLoading": true,
|
||||
"description": "Gateway scheduler: reminders, delayed self-wakeups, loops, recurring work, event watchers. Never exec sleep/poll as timer.\n\nACTIONS: status | list [includeDisabled,limit?,offset?] (use nextOffset for the next page) | get jobId | add job | update jobId patch | remove jobId | run jobId (runMode \"force\"=now) | runs jobId = history | next_check in:\"30m\" (own paced run only) | wake text mode?:\"now\"|\"next-heartbeat\"(default) nudges a caller-owned lane (sessionKey/agentId to pick another).\n\nADD: {name?,schedule,payload,sessionTarget?,pacing?,trigger?,delivery?,enabled?}. Required: schedule+payload.\n\nSCHEDULE:\n- {kind:\"at\",at:\"ISO-8601\"} one-shot; no tz=UTC; auto-deletes after run.\n- {kind:\"every\",everyMs}.\n- {kind:\"cron\",expr,tz?:\"IANA\"}: expr is wall time in tz; never pre-convert to UTC; no tz=gateway host local. 18:00 Shanghai => {expr:\"0 18 * * *\",tz:\"Asia/Shanghai\"}.\n- {kind:\"stream\",command:[argv],mode?:\"line\"|\"match\",match?}: fires on supervised process output; needs cron.triggers.enabled.\n\nTARGET+PAYLOAD:\n- \"current\" (agentTurn default) = this conversation: run carries this chat's context, result lands here. Self-wakeup/\"continue later\"/loop = at|every + agentTurn + current.\n- \"isolated\" = fresh detached session (shows in `openclaw tasks`); standalone background work.\n- \"main\" = heartbeat lane; payload {kind:\"systemEvent\",text} (systemEvent default target).\n- \"session:<key>\" = named session.\n- agentTurn {kind:\"agentTurn\",message,model?,thinking?,timeoutSeconds?}; timeoutSeconds 0=none.\n- Inherited configured MCP authority includes only model-callable tools; interactive app-view-only capabilities are excluded from headless jobs.\n- script {kind:\"script\",script,timeoutSeconds?,toolBudget?}: main|isolated only; needs cron.triggers.enabled.\n\nPACED LOOP: recurring job + pacing{min?,max?} durations (\"15m\",\"4h\"; at least one). Inside its run, job calls next_check in:\"<dur>\" to set the next delay (clamped to bounds, measured from run end; failed runs keep normal backoff). Adaptive polling: tighten when active, back off when quiet.\n\nTRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call(\"exec\",{command:\"...\"}).\n\nDELIVERY {mode:\"none\"|\"announce\"|\"webhook\",channel?,to?,threadId?,bestEffort?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:\"none\". webhook posts finished-run event to URL in `to`.\n\nJob wakeMode (main jobs): \"now\"(default)|\"next-heartbeat\". Restricted automation-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.",
|
||||
"description": "Gateway scheduler: reminders, delayed self-wakeups, loops, recurring work, event watchers. Never exec sleep/poll as timer.\n\nACTIONS: status | list [includeDisabled,limit?,offset?] (use nextOffset for the next page) | get jobId | add job | update jobId patch | remove jobId | run jobId (runMode \"force\"=now) | runs jobId = history | next_check in:\"30m\" (own paced run only) | wake text mode?:\"now\"|\"next-heartbeat\"(default) nudges a caller-owned lane (sessionKey/agentId to pick another).\n\nADD: {name?,schedule,payload,sessionTarget?,pacing?,trigger?,delivery?,enabled?}. Required: schedule+payload.\n\nSCHEDULE:\n- {kind:\"at\",at:\"ISO-8601\"} one-shot; no tz=UTC; auto-deletes after run.\n- {kind:\"every\",everyMs}.\n- {kind:\"cron\",expr,tz?:\"IANA\"}: expr is wall time in tz; never pre-convert to UTC; no tz=gateway host local. 18:00 Shanghai => {expr:\"0 18 * * *\",tz:\"Asia/Shanghai\"}.\n- {kind:\"stream\",command:[argv],mode?:\"line\"|\"match\",match?}: fires on supervised process output; needs cron.triggers.enabled.\n\nTARGET+PAYLOAD:\n- \"current\" (agentTurn default) = this conversation: run carries this chat's context, result lands here. Self-wakeup/\"continue later\"/loop = at|every + agentTurn + current.\n- \"isolated\" = fresh detached session (shows in `openclaw tasks`); standalone background work.\n- \"main\" = heartbeat lane; payload {kind:\"systemEvent\",text} (systemEvent default target).\n- \"session:<key>\" = named session.\n- agentTurn {kind:\"agentTurn\",message,model?,thinking?,timeoutSeconds?}; timeoutSeconds 0=none.\n- Inherited configured MCP authority includes only model-callable tools; interactive app-view-only capabilities are excluded from headless jobs.\n- script {kind:\"script\",script,timeoutSeconds?,toolBudget?}: main|isolated only; needs cron.triggers.enabled.\n\nPACED LOOP: recurring job + pacing{min?,max?} durations (\"15m\",\"4h\"; at least one). Inside its run, job calls next_check in:\"<dur>\" to set the next delay (clamped to bounds, measured from run end; failed runs keep normal backoff). Adaptive polling: tighten when active, back off when quiet.\n\nTRIGGER (condition watcher on every/cron): {script,once?}; needs cron.triggers.enabled — if off, say so; never model-poll instead. Quiet headless check, no model; 30s/5 tool calls/16KB state. Read frozen trigger.state, return json({fire,message?,state?}) with NEW state; dedupe via state, never memory. fire:false saves state only. fire:true runs payload; message is that run's entire context — self-contained. Fire on failures/timeouts too; success-only watchers look healthy when broken. Script stays read-only; actions belong in payload. once:true disables after first fire. Code Mode: await tools.call(\"exec\",{command:\"...\"}).\n\nDELIVERY {mode:\"none\"|\"announce\"|\"webhook\",channel?,to?,threadId?,bestEffort?,completionDestination?}: where detached run output goes. Omitted=announce (current=>this chat; isolated=>last route; set channel/to for a specific chat — no messaging tool inside the run). Silent watcher=>mode:\"none\". webhook posts finished-run event to URL in `to`. To keep announce delivery and also POST completion, use mode:\"announce\" with completionDestination:{mode:\"webhook\",to:\"https://...\"}.\n\nJob wakeMode (main jobs): \"now\"(default)|\"next-heartbeat\". Restricted automation-run sessions: self status/list/get/runs/remove + own next_check only. failureAlert {...}|false disables. jobId canonical (id=compat). contextMessages 0-10 embeds recent chat lines into reminder text.",
|
||||
"inputSchema": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
@@ -343,6 +343,23 @@
|
||||
"description": "Delivery channel",
|
||||
"type": "string"
|
||||
},
|
||||
"completionDestination": {
|
||||
"additionalProperties": true,
|
||||
"description": "Additional completion webhook; requires delivery.mode=announce",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"const": "webhook",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"description": "Completion webhook target; only valid with delivery.mode=announce",
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["mode", "to"],
|
||||
"type": "object"
|
||||
},
|
||||
"failureDestination": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
@@ -542,6 +559,7 @@
|
||||
"properties": {
|
||||
"anchorMs": {
|
||||
"description": "Start anchor ms (kind=every)",
|
||||
"maximum": 8640000000000000,
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
@@ -568,6 +586,7 @@
|
||||
},
|
||||
"everyMs": {
|
||||
"description": "Interval ms (kind=every)",
|
||||
"maximum": 8640000000000000,
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
},
|
||||
@@ -594,6 +613,7 @@
|
||||
},
|
||||
"staggerMs": {
|
||||
"description": "Jitter ms (kind=cron)",
|
||||
"maximum": 8640000000000000,
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
@@ -706,6 +726,31 @@
|
||||
],
|
||||
"description": "Delivery channel, or null to clear"
|
||||
},
|
||||
"completionDestination": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"description": "Additional completion webhook; requires delivery.mode=announce",
|
||||
"properties": {
|
||||
"mode": {
|
||||
"const": "webhook",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"description": "Completion webhook target; only valid with delivery.mode=announce",
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["mode", "to"],
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Completion webhook destination; requires delivery.mode=announce; null clears."
|
||||
},
|
||||
"failureDestination": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -969,6 +1014,7 @@
|
||||
"properties": {
|
||||
"anchorMs": {
|
||||
"description": "Start anchor ms (kind=every)",
|
||||
"maximum": 8640000000000000,
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
@@ -995,6 +1041,7 @@
|
||||
},
|
||||
"everyMs": {
|
||||
"description": "Interval ms (kind=every)",
|
||||
"maximum": 8640000000000000,
|
||||
"minimum": 1,
|
||||
"type": "integer"
|
||||
},
|
||||
@@ -1021,6 +1068,7 @@
|
||||
},
|
||||
"staggerMs": {
|
||||
"description": "Jitter ms (kind=cron)",
|
||||
"maximum": 8640000000000000,
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
|
||||
Vendored
+4
-4
@@ -227,8 +227,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
|
||||
"roughTokens": 0
|
||||
},
|
||||
"dynamicToolsJson": {
|
||||
"chars": 62876,
|
||||
"roughTokens": 15719
|
||||
"chars": 65442,
|
||||
"roughTokens": 16361
|
||||
},
|
||||
"openClawDeveloperInstructions": {
|
||||
"chars": 4519,
|
||||
@@ -239,8 +239,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
|
||||
"roughTokens": 7226
|
||||
},
|
||||
"totalWithDynamicToolsJson": {
|
||||
"chars": 91780,
|
||||
"roughTokens": 22945
|
||||
"chars": 94346,
|
||||
"roughTokens": 23587
|
||||
},
|
||||
"userInputText": {
|
||||
"chars": 1300,
|
||||
|
||||
test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md
Vendored
+4
-4
@@ -227,8 +227,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
|
||||
"roughTokens": 0
|
||||
},
|
||||
"dynamicToolsJson": {
|
||||
"chars": 62568,
|
||||
"roughTokens": 15642
|
||||
"chars": 65134,
|
||||
"roughTokens": 16284
|
||||
},
|
||||
"openClawDeveloperInstructions": {
|
||||
"chars": 3410,
|
||||
@@ -239,8 +239,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
|
||||
"roughTokens": 6856
|
||||
},
|
||||
"totalWithDynamicToolsJson": {
|
||||
"chars": 89992,
|
||||
"roughTokens": 22498
|
||||
"chars": 92558,
|
||||
"roughTokens": 23140
|
||||
},
|
||||
"userInputText": {
|
||||
"chars": 929,
|
||||
|
||||
Vendored
+4
-4
@@ -222,8 +222,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
|
||||
"roughTokens": 0
|
||||
},
|
||||
"dynamicToolsJson": {
|
||||
"chars": 64102,
|
||||
"roughTokens": 16026
|
||||
"chars": 66668,
|
||||
"roughTokens": 16667
|
||||
},
|
||||
"openClawDeveloperInstructions": {
|
||||
"chars": 3410,
|
||||
@@ -234,8 +234,8 @@ This is the deterministic model-bound layer stack OpenClaw can snapshot for the
|
||||
"roughTokens": 6960
|
||||
},
|
||||
"totalWithDynamicToolsJson": {
|
||||
"chars": 91942,
|
||||
"roughTokens": 22986
|
||||
"chars": 94508,
|
||||
"roughTokens": 23627
|
||||
},
|
||||
"userInputText": {
|
||||
"chars": 1271,
|
||||
|
||||
@@ -31,7 +31,7 @@ describe("audit-seams cron seam classification", () => {
|
||||
|
||||
it("detects scheduler-state seams in cron service orchestration", () => {
|
||||
const source = `
|
||||
import { recomputeNextRuns, computeJobNextRunAtMs } from "./jobs.js";
|
||||
import { recomputeNextRuns, computeJobNextRunAtMs } from "./jobs-scheduling.js";
|
||||
import { ensureLoaded, persist } from "./store.js";
|
||||
import { armTimer, runMissedJobs } from "./timer.js";
|
||||
|
||||
|
||||
Reference in New Issue
Block a user