fix(gateway): prevent restart replay after final delivery (#121908)

Re-lands the reverted #121507 on the current custody contracts. The reply dispatcher now owns direct-send pending-final custody: claim before provider I/O, terminal settlement for delivered/suppressed/failed outcomes, proven no-send stays replayable, ambiguous evidence fails closed — so Gateway restarts can no longer duplicate an already-accepted final reply.

Proof: ClawSweeper local review clean, exact-head ci-gate green, live Telegram E2E (one turn, one final, no duplicates).

Co-authored-by: Ayaan Zaidi <hi@obviy.us>
This commit is contained in:
Ayaan Zaidi
2026-08-11 13:09:59 +05:30
committed by GitHub
parent cc99646601
commit 210aca6de3
73 changed files with 2721 additions and 832 deletions
File diff suppressed because it is too large Load Diff
@@ -323,6 +323,7 @@ export async function dispatchDiscordComponentEvent(params: {
chunkMode: resolveChunkMode(ctx.cfg, "discord", accountId),
mediaLocalRoots,
kind: info.kind,
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
});
if (result.visibleReplySent) {
replyReference.markSent();
@@ -271,7 +271,10 @@ async function processDiscordMessageInner(
const deliverDiscordPayload = async (
payload: ReplyPayload,
info: { kind: ReplyDispatchKind },
info: {
kind: ReplyDispatchKind;
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
},
options?: {
allowFallbackOnlyToolWarning?: boolean;
allowProgressBlock?: boolean;
@@ -328,6 +331,7 @@ async function processDiscordMessageInner(
threadBindings,
mediaLocalRoots,
kind: "block",
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
});
if (result.visibleReplySent) {
replyReference.markSent();
@@ -484,6 +488,7 @@ async function processDiscordMessageInner(
mediaLocalRoots,
allowedMentions,
kind: info.kind,
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
});
return deliveryResult.visibleReplySent;
},
@@ -542,6 +547,7 @@ async function processDiscordMessageInner(
threadBindings,
mediaLocalRoots,
kind: info.kind,
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
});
if (!result.visibleReplySent) {
return result;
@@ -229,13 +229,16 @@ export async function deliverDiscordReply(params: {
mediaLocalRoots?: readonly string[];
allowedMentions?: DiscordAllowedMentions;
kind: "tool" | "block" | "final";
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
}) {
void params.runtime;
const delivery = resolveDiscordDeliveryOptions(params);
const payloads = sanitizeDiscordFrontChannelReplyPayloads(params.replies, {
kind: params.kind,
}).map(formatDiscordReasoningPayload);
})
.map(formatDiscordReasoningPayload)
.map((payload) => params.bindPendingFinalDelivery?.(payload) ?? payload);
if (payloads.length === 0) {
return {
visibleReplySent: false,
@@ -86,9 +86,7 @@ export function sanitizeDiscordFrontChannelReplyPayloads(
: sanitizeDiscordFrontChannelText(payload.text)
: payload.text;
const nextPayload =
safeText === payload.text
? payload
: ({ ...payload, text: safeText || undefined } as ReplyPayload);
safeText === payload.text ? payload : { ...payload, text: safeText || undefined };
const nextParts = resolveSendableOutboundReplyParts(nextPayload);
if (!nextParts.hasContent && !hasNonTextReplyPayloadContent(nextPayload)) {
continue;
+29 -11
View File
@@ -65,6 +65,7 @@ async function maybeSendDiscordWebhookText(params: {
accountId?: string | null;
identity?: OutboundIdentity;
replyToId?: string | null;
onPlatformSendDispatch?: () => Promise<void>;
}): Promise<{ messageId: string; channelId: string } | null> {
if (params.threadId == null) {
return null;
@@ -96,6 +97,7 @@ async function maybeSendDiscordWebhookText(params: {
replyTo: params.replyToId ?? undefined,
username: persona.username,
avatarUrl: persona.avatarUrl,
onPlatformSendDispatch: params.onPlatformSendDispatch,
});
return result;
}
@@ -130,6 +132,7 @@ async function resolveDiscordOutboundMessageSend(params: DiscordOutboundMessageC
await params.onDeliveryResult?.(attachChannelToResult("discord", result));
}
: undefined,
onPlatformSendDispatch: params.onPlatformSendDispatch,
},
};
}
@@ -173,16 +176,29 @@ export const discordOutbound: ChannelOutboundAdapter = {
channel: "discord",
sendText: async (ctx) => {
if (!ctx.silent) {
const webhookResult = await maybeSendDiscordWebhookText({
cfg: ctx.cfg,
text: ctx.text,
threadId: ctx.threadId,
accountId: ctx.accountId,
identity: ctx.identity,
replyToId: ctx.replyToId,
}).catch(() => null);
if (webhookResult) {
return webhookResult;
let webhookSelected = false;
try {
const webhookResult = await maybeSendDiscordWebhookText({
cfg: ctx.cfg,
text: ctx.text,
threadId: ctx.threadId,
accountId: ctx.accountId,
identity: ctx.identity,
replyToId: ctx.replyToId,
onPlatformSendDispatch: ctx.onPlatformSendDispatch
? async () => {
webhookSelected = true;
await ctx.onPlatformSendDispatch?.();
}
: undefined,
});
if (webhookResult) {
return webhookResult;
}
} catch (error) {
if (webhookSelected) {
throw error;
}
}
}
const { send, target, options } = await resolveDiscordOutboundMessageSend(ctx);
@@ -202,6 +218,7 @@ export const discordOutbound: ChannelOutboundAdapter = {
mediaAccess: ctx.mediaAccess,
mediaLocalRoots: ctx.mediaLocalRoots,
mediaReadFile: ctx.mediaReadFile,
onPlatformSendDispatch: ctx.onPlatformSendDispatch,
});
}
const mediaOptions = {
@@ -235,13 +252,14 @@ export const discordOutbound: ChannelOutboundAdapter = {
}
return await send(target, ctx.text, mediaOptions);
},
sendPoll: async ({ cfg, to, poll, accountId, threadId, silent }) =>
sendPoll: async ({ cfg, to, poll, accountId, threadId, silent, onPlatformSendDispatch }) =>
await (
await loadDiscordSendRuntime()
).sendPollDiscord(resolveDiscordOutboundTarget({ to, threadId }), poll, {
accountId: accountId ?? undefined,
silent: silent ?? undefined,
cfg,
onPlatformSendDispatch,
}),
}),
afterDeliverPayload: async ({ cfg, target, payload, results }) => {
@@ -57,6 +57,7 @@ function resolveDiscordDeliveryOptions(
accountId: ctx.accountId ?? undefined,
silent: ctx.silent ?? undefined,
cfg: ctx.cfg,
onPlatformSendDispatch: ctx.onPlatformSendDispatch,
};
}
@@ -172,6 +172,7 @@ type DiscordComponentSendOpts = {
allowedMentions?: DiscordAllowedMentions;
/** Persist the concrete platform send before component bookkeeping can fail. */
onDeliveryResult?: (result: DiscordSendResult) => Promise<void> | void;
onPlatformSendDispatch?: () => Promise<void>;
};
export function registerBuiltDiscordComponentMessage(params: {
@@ -291,6 +292,7 @@ export async function sendDiscordComponentMessage(
tableMode: opts.tableMode,
chunkMode: opts.chunkMode,
onDeliveryResult: opts.onDeliveryResult,
onPlatformSendDispatch: opts.onPlatformSendDispatch,
...(opts.suppressEmbeds === undefined ? {} : { suppressEmbeds: opts.suppressEmbeds }),
});
}
@@ -321,6 +323,7 @@ export async function sendDiscordComponentMessage(
let result: { id: string; channel_id: string };
try {
await opts.onPlatformSendDispatch?.();
result = (await request(
() =>
createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, {
+12
View File
@@ -70,6 +70,8 @@ type DiscordSendOpts = {
allowedMentions?: DiscordAllowedMentions;
/** Persist each concrete platform send before any later chunk can fail. */
onDeliveryResult?: (result: DiscordSendResult) => Promise<void> | void;
/** @internal Refresh durable custody immediately before Discord REST I/O. */
onPlatformSendDispatch?: () => Promise<void>;
};
type DiscordClientRequest = ReturnType<typeof createDiscordClient>["request"];
@@ -92,6 +94,7 @@ async function sendDiscordThreadTextChunks(params: {
suppressEmbeds?: boolean;
allowedMentions?: DiscordAllowedMentions;
onResult?: DiscordSendProgress;
onPlatformSendDispatch?: () => Promise<void>;
}): Promise<void> {
for (const chunk of params.chunks) {
await sendDiscordText({
@@ -106,6 +109,7 @@ async function sendDiscordThreadTextChunks(params: {
allowedMentions: params.allowedMentions,
maxChars: params.maxChars,
onResult: params.onResult,
onPlatformSendDispatch: params.onPlatformSendDispatch,
});
}
}
@@ -238,6 +242,7 @@ export async function sendMessageDiscord(
});
let threadRes: { id: string; message?: { id: string; channel_id: string } };
try {
await opts.onPlatformSendDispatch?.();
threadRes = (await request(
() =>
createThread<{ id: string; message?: { id: string; channel_id: string } }>(
@@ -309,6 +314,7 @@ export async function sendMessageDiscord(
allowedMentions: opts.allowedMentions,
maxChars: textLimit,
onResult: reportThreadResult,
onPlatformSendDispatch: opts.onPlatformSendDispatch,
});
await sendDiscordThreadTextChunks({
rest,
@@ -322,6 +328,7 @@ export async function sendMessageDiscord(
suppressEmbeds,
allowedMentions: opts.allowedMentions,
onResult: reportThreadResult,
onPlatformSendDispatch: opts.onPlatformSendDispatch,
});
} else {
await sendDiscordThreadTextChunks({
@@ -336,6 +343,7 @@ export async function sendMessageDiscord(
suppressEmbeds,
allowedMentions: opts.allowedMentions,
onResult: reportThreadResult,
onPlatformSendDispatch: opts.onPlatformSendDispatch,
});
}
} catch (err) {
@@ -391,6 +399,7 @@ export async function sendMessageDiscord(
allowedMentions: opts.allowedMentions,
maxChars: textLimit,
onResult: reportResult,
onPlatformSendDispatch: opts.onPlatformSendDispatch,
});
} else {
result = await sendDiscordText({
@@ -408,6 +417,7 @@ export async function sendMessageDiscord(
allowedMentions: opts.allowedMentions,
maxChars: textLimit,
onResult: reportResult,
onPlatformSendDispatch: opts.onPlatformSendDispatch,
});
}
} catch (err) {
@@ -447,6 +457,7 @@ export async function sendStickerDiscord(
enforce_nonce: true,
...(flags ? { flags } : {}),
};
await opts.onPlatformSendDispatch?.();
const res = (await request(
() => createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, { body }),
"sticker",
@@ -474,6 +485,7 @@ export async function sendPollDiscord(
enforce_nonce: true,
...(flags ? { flags } : {}),
};
await opts.onPlatformSendDispatch?.();
const res = (await request(
() => createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, { body }),
"poll",
+6
View File
@@ -325,6 +325,7 @@ type DiscordTextSendParams = {
suppressEmbeds?: boolean;
maxChars?: number;
onResult?: DiscordSendProgress;
onPlatformSendDispatch?: () => Promise<void>;
};
async function sendDiscordText(params: DiscordTextSendParams) {
@@ -343,6 +344,7 @@ async function sendDiscordText(params: DiscordTextSendParams) {
suppressEmbeds,
maxChars,
onResult,
onPlatformSendDispatch,
} = params;
if (!text.trim()) {
throw new Error("Message must be non-empty for Discord sends");
@@ -369,6 +371,7 @@ async function sendDiscordText(params: DiscordTextSendParams) {
flags,
replyTo: chunkReplyTo,
});
await onPlatformSendDispatch?.();
const result = (await request(
() => createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, { body }),
"text",
@@ -429,6 +432,7 @@ async function sendDiscordMedia(params: DiscordMediaSendParams) {
suppressEmbeds,
maxChars,
onResult,
onPlatformSendDispatch,
} = params;
const media = await loadWebMedia(
mediaUrl,
@@ -472,6 +476,7 @@ async function sendDiscordMedia(params: DiscordMediaSendParams) {
});
let res: { id: string; channel_id: string };
try {
await onPlatformSendDispatch?.();
res = (await request(
() => createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, { body }),
"media",
@@ -496,6 +501,7 @@ async function sendDiscordMedia(params: DiscordMediaSendParams) {
allowedMentions,
maxChars,
onResult,
onPlatformSendDispatch,
});
}
await onResult?.(res, "media", reply?.messageId);
+2
View File
@@ -38,6 +38,7 @@ type VoiceMessageOpts = Pick<
| "mediaAccess"
| "mediaLocalRoots"
| "mediaReadFile"
| "onPlatformSendDispatch"
>;
function toDiscordSendResult(
@@ -122,6 +123,7 @@ export async function sendVoiceMessageDiscord(
const metadata = await getVoiceMessageMetadata(oggPath);
const audioBuffer = await fs.readFile(oggPath);
await opts.onPlatformSendDispatch?.();
const result = await sendDiscordVoiceMessage(
rest,
channelId,
+2
View File
@@ -40,6 +40,7 @@ type DiscordWebhookSendOpts = {
username?: string;
avatarUrl?: string;
wait?: boolean;
onPlatformSendDispatch?: () => Promise<void>;
};
function resolveWebhookExecutionUrl(params: {
@@ -154,6 +155,7 @@ export async function sendWebhookMessageDiscord(
try {
const response = await request(
async () => {
await opts.onPlatformSendDispatch?.();
const attemptResponse = await (proxyFetch ?? fetch)(url, {
method: "POST",
headers: {
@@ -208,7 +208,10 @@ export function createTelegramDeliveryController(params: {
) {
return payload;
}
return { ...payload, replyToId: implicitQuoteReplyTargetId };
return {
...payload,
replyToId: implicitQuoteReplyTargetId,
};
};
const usesNativeTelegramQuote = (payload: ReplyPayload): boolean =>
params.replyQuoteText != null ||
@@ -223,6 +226,8 @@ export function createTelegramDeliveryController(params: {
mirrorTranscript?: boolean;
promptContextSequence?: TelegramPromptContextProjectionSequence;
textMode?: "html";
onPlatformSendDispatch?: () => Promise<void>;
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
},
) => {
if (params.isDispatchSuperseded()) {
@@ -253,10 +258,13 @@ export function createTelegramDeliveryController(params: {
)
: undefined,
);
const effectivePayload = withTelegramPromptContextSource(
const projectedPayload = withTelegramPromptContextSource(
deliverablePayload,
projectionSequence.source,
);
const effectivePayload = options?.bindPendingFinalDelivery
? options.bindPendingFinalDelivery(projectedPayload)
: projectedPayload;
const silent =
options?.silent ??
(params.telegramCfg.silentErrorReplies === true && payload.isError === true);
@@ -316,6 +324,7 @@ export function createTelegramDeliveryController(params: {
silent,
mediaLoader: params.telegramDeps.loadWebMedia,
promptContextSequence: projectionSequence,
onPlatformSendDispatch: options?.onPlatformSendDispatch,
...(options?.textMode ? { textMode: options.textMode } : {}),
});
if (!result.delivered) {
@@ -432,6 +441,8 @@ export function createTelegramDeliveryController(params: {
payload: ReplyPayload,
text: string,
promptContextSequence: TelegramPromptContextProjectionSequence,
onPlatformSendDispatch?: () => Promise<void>,
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T,
): Promise<LaneDeliveryResult> => {
const afterAcceptedDraft = params.draft.answerLane.stream?.hasConsumedReplyTarget?.() === true;
if (payload.isError === true) {
@@ -441,6 +452,8 @@ export function createTelegramDeliveryController(params: {
afterAcceptedDraft,
durable: true,
promptContextSequence,
onPlatformSendDispatch,
bindPendingFinalDelivery,
});
if (!delivered) {
return { kind: "skipped" };
@@ -454,6 +467,8 @@ export function createTelegramDeliveryController(params: {
afterAcceptedDraft,
durable: true,
promptContextSequence,
onPlatformSendDispatch,
bindPendingFinalDelivery,
});
if (barLine) {
await params.progress.applyCollapseSummary(barLine, postCosmeticSummaryBar);
@@ -472,6 +487,8 @@ export function createTelegramDeliveryController(params: {
answerPayload: ReplyPayload,
text: string,
buttons?: TelegramInlineButtons,
onPlatformSendDispatch?: () => Promise<void>,
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T,
): Promise<LaneDeliveryResult> => {
const transcriptFinal = await resolveCurrentTurnTranscriptFinal();
const finalText = await resolveTranscriptBackedChannelFinalText({
@@ -491,6 +508,8 @@ export function createTelegramDeliveryController(params: {
answerPayload,
finalText,
promptContextSequence,
onPlatformSendDispatch,
bindPendingFinalDelivery,
);
} else {
if (isFollowUp) {
@@ -506,6 +525,8 @@ export function createTelegramDeliveryController(params: {
buttons,
allowStream: !usesNativeTelegramQuote(answerPayload),
promptContextSequence,
onPlatformSendDispatch,
bindPendingFinalDelivery,
});
if (!isFollowUp && result.kind !== "skipped") {
params.progress.markFinalDelivered();
@@ -9,8 +9,8 @@ import {
isFastModeAutoProgressPayload,
isReplyPayloadNonTerminalToolErrorWarning,
resolveSendableOutboundReplyParts,
type ReplyPayload,
} from "openclaw/plugin-sdk/reply-payload";
import type { ReplyPayload } from "openclaw/plugin-sdk/reply-payload";
import type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";
import { danger } from "openclaw/plugin-sdk/runtime-env";
import type { TelegramBotDeps } from "./bot-deps.js";
@@ -105,6 +105,8 @@ export function createTelegramReplyDelivery(params: {
| {
promise: Promise<{ visibleReplySent: boolean }>;
visibleReplySent: boolean;
onPlatformSendDispatch?: () => Promise<void>;
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
resolve: (result: { visibleReplySent: boolean }) => void;
reject: (error: unknown) => void;
}
@@ -144,6 +146,8 @@ export function createTelegramReplyDelivery(params: {
buffered.payload,
buffered.text,
resolvePayloadTelegramInlineButtons(buffered.payload),
settlement?.onPlatformSendDispatch,
settlement?.bindPendingFinalDelivery,
);
if (settlement) {
settlement.resolve({
@@ -271,6 +275,8 @@ export function createTelegramReplyDelivery(params: {
bufferedFinalSettlement = {
promise: finalization,
visibleReplySent: blockDelivered,
onPlatformSendDispatch: info.onPlatformSendDispatch,
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
resolve: resolveFinalization,
reject: rejectFinalization,
};
@@ -381,6 +387,8 @@ export function createTelegramReplyDelivery(params: {
effectivePayload,
segment.update.text,
telegramButtons,
info.onPlatformSendDispatch,
info.bindPendingFinalDelivery,
)
: await params.delivery.deliverLaneText({
laneName: segment.lane,
@@ -389,6 +397,8 @@ export function createTelegramReplyDelivery(params: {
infoKind: info.kind,
buttons: telegramButtons,
allowStream: !isDurableProgressCommentary,
onPlatformSendDispatch: info.onPlatformSendDispatch,
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
});
if (
segment.lane === "answer" &&
@@ -440,6 +450,8 @@ export function createTelegramReplyDelivery(params: {
: effectivePayload;
delivered = await params.delivery.sendPayload(payloadWithoutReasoning, {
durable: info.kind === "final",
onPlatformSendDispatch: info.onPlatformSendDispatch,
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
});
}
if (info.kind === "final" && delivered) {
@@ -463,6 +475,8 @@ export function createTelegramReplyDelivery(params: {
}
const delivered = await params.delivery.sendPayload(effectivePayload, {
durable: info.kind === "final",
onPlatformSendDispatch: info.onPlatformSendDispatch,
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
});
if (info.kind === "final" && delivered) {
params.progress.markFinalDelivered();
@@ -1777,7 +1777,7 @@ export const registerTelegramNativeCommands = ({
},
},
delivery: {
deliverWithProviderMessageSending: async (payload) => {
deliverWithProviderMessageSending: async (payload, info) => {
if (
shouldSuppressLocalTelegramExecApprovalPrompt({
cfg: runtimeCfg,
@@ -1802,6 +1802,7 @@ export const registerTelegramNativeCommands = ({
],
...deliveryBaseOptions,
silent: runtimeTelegramCfg.silentErrorReplies === true && payload.isError === true,
onPlatformSendDispatch: info.onPlatformSendDispatch,
});
if (result.delivered) {
deliveryState.delivered = true;
@@ -71,16 +71,21 @@ export function createTelegramNativeCommandTestDeps(
): { dispatchChannelInboundTurn: DispatchChannelInboundTurn } {
return {
dispatchChannelInboundTurn: async (plan) => {
const delivery = plan.delivery;
const dispatchResult = await dispatchReply({
ctx: plan.ctxPayload,
cfg: plan.cfg,
dispatcherOptions: {
...plan.dispatcherOptions,
deliver:
"deliverWithProviderMessageSending" in plan.delivery
? plan.delivery.deliverWithProviderMessageSending
: plan.delivery.deliver,
onError: plan.delivery.onError,
"deliverWithProviderMessageSending" in delivery
? (payload, info) =>
delivery.deliverWithProviderMessageSending(payload, {
...info,
onPlatformSendDispatch: info.onPlatformSendDispatch ?? (async () => undefined),
})
: delivery.deliver,
onError: delivery.onError,
},
replyOptions: plan.replyOptions,
});
+5 -1
View File
@@ -38,7 +38,11 @@ export async function runTelegramChannelInboundEventWithHarness(
cfg: plan.cfg,
dispatcherOptions: {
...plan.dispatcherOptions,
deliver: plan.delivery.deliverWithProviderMessageSending,
deliver: (payload, info) =>
plan.delivery.deliverWithProviderMessageSending(payload, {
...info,
onPlatformSendDispatch: info.onPlatformSendDispatch ?? (async () => undefined),
}),
onError: plan.delivery.onError,
},
toolsAllow: plan.toolsAllow,
@@ -254,6 +254,7 @@ async function deliverTextReply(params: {
progress: DeliveryProgress;
recordMessageId: (messageId: number) => void;
quoteOnlyOnFirstChunk?: boolean;
onPlatformSendDispatch?: () => Promise<void>;
}): Promise<number | undefined> {
let firstDeliveredMessageId: number | undefined;
const chunks = filterEmptyTelegramTextChunks(params.chunkText(params.text));
@@ -278,6 +279,7 @@ async function deliverTextReply(params: {
markDelivered,
sendChunk: async ({ chunk, isFirstChunk, replyToMessageId, replyMarkup, replyQuoteText }) => {
const includeQuoteMetadata = params.quoteOnlyOnFirstChunk !== true || isFirstChunk;
await params.onPlatformSendDispatch?.();
const messageId = await sendTelegramText(
params.bot,
params.chatId,
@@ -351,6 +353,7 @@ async function deliverMediaReply(params: {
progress: DeliveryProgress;
recordMessageId: (messageId: number) => void;
textMode?: "html";
onPlatformSendDispatch?: () => Promise<void>;
}): Promise<{ firstDeliveredMessageId?: number; visibleFallbackText?: string }> {
let firstDeliveredMessageId: number | undefined;
let visibleFallbackText: string | undefined;
@@ -372,6 +375,7 @@ async function deliverMediaReply(params: {
plainCaption?: string;
shouldLog?: (err: unknown) => boolean;
}) => {
await params.onPlatformSendDispatch?.();
const delivery = await sendTelegramCaptionedMediaWithFallback({
operation: options.sender.operation,
requestParams: options.requestParams,
@@ -517,6 +521,7 @@ async function deliverMediaReply(params: {
progress: createVoiceFallbackProgress(),
recordMessageId: params.recordMessageId,
quoteOnlyOnFirstChunk: true,
onPlatformSendDispatch: params.onPlatformSendDispatch,
});
await params.onVoiceRecording?.();
@@ -612,6 +617,7 @@ async function deliverMediaReply(params: {
replyToMode: params.replyToMode,
progress: params.progress,
recordMessageId: params.recordMessageId,
onPlatformSendDispatch: params.onPlatformSendDispatch,
});
if (followUpMessageId === undefined) {
visibleFallbackText = firstDeliveredCaption ?? "";
@@ -783,6 +789,8 @@ export async function deliverReplies(params: {
promptContextSequence?: TelegramPromptContextProjectionSequence;
/** Text is already prepared Telegram HTML and must not be parsed as Markdown again. */
textMode?: "html";
/** @internal Claim delivery custody immediately before Telegram Bot API I/O. */
onPlatformSendDispatch?: () => Promise<void>;
}): Promise<{
delivered: boolean;
}> {
@@ -930,6 +938,7 @@ export async function deliverReplies(params: {
);
let firstDeliveredMessageId: number | undefined;
if (reactionEmoji && typeof replyToId === "number") {
await params.onPlatformSendDispatch?.();
const reactionResult = await reactMessageTelegram(params.chatId, replyToId, reactionEmoji, {
cfg: params.cfg ?? { channels: { telegram: { botToken: params.token } } },
token: params.token,
@@ -966,6 +975,7 @@ export async function deliverReplies(params: {
replyToMode: params.replyToMode,
progress,
recordMessageId,
onPlatformSendDispatch: params.onPlatformSendDispatch,
});
} else if (mediaList.length > 0) {
const mediaDelivery = await deliverMediaReply({
@@ -993,6 +1003,7 @@ export async function deliverReplies(params: {
replyToMode: params.replyToMode,
progress,
recordMessageId,
onPlatformSendDispatch: params.onPlatformSendDispatch,
...(params.textMode ? { textMode: params.textMode } : {}),
});
firstDeliveredMessageId = mediaDelivery.firstDeliveredMessageId;
+14 -3
View File
@@ -61,7 +61,7 @@ const MAX_PREVIEW_FLOOD_SUSPEND_MS = 60_000;
const MIN_PREVIEW_DWELL_MS = 4_000;
export type TelegramDraftStream = {
update: (text: string) => void;
update: (text: string, options?: { onPlatformSendDispatch?: () => Promise<void> }) => void;
updateLazy: (resolveText: () => string | undefined) => void;
updatePreview: (preview: TelegramDraftPreview) => void;
flush: () => Promise<void>;
@@ -323,6 +323,7 @@ export function createTelegramDraftStream(params: {
let lastDeliveredText = "";
let lastRequestedText = "";
let lastRequestedPreview: TelegramDraftPreview | undefined;
let pendingPlatformSendDispatch: (() => Promise<void>) | undefined;
let generation = 0;
let finalPagePlan: { pages: PlannedTelegramDraftPage[]; nextPageIndex: number } | undefined;
// Generations whose in-flight FIRST send was superseded by a reposition
@@ -450,6 +451,10 @@ export function createTelegramDraftStream(params: {
page: PlannedTelegramDraftPage,
sendGeneration: number,
): Promise<boolean> => {
if (pendingPlatformSendDispatch) {
await pendingPlatformSendDispatch();
pendingPlatformSendDispatch = undefined;
}
const targetMessageId = streamMessageId;
if (typeof targetMessageId === "number") {
streamVisibleSinceMs ??= Date.now();
@@ -771,12 +776,17 @@ export function createTelegramDraftStream(params: {
throwTerminalDeliveryError();
};
const requestDraftUpdate = (text: string, preview?: TelegramDraftPreview) => {
const requestDraftUpdate = (
text: string,
preview?: TelegramDraftPreview,
onPlatformSendDispatch?: () => Promise<void>,
) => {
if (streamState.stopped || streamState.final) {
return;
}
lastRequestedPreview = preview;
lastRequestedText = text;
pendingPlatformSendDispatch = onPlatformSendDispatch;
updateDraft(text);
};
@@ -849,6 +859,7 @@ export function createTelegramDraftStream(params: {
streamState.final = true;
observeCurrentProviderMessage();
await drainProviderMessageObservations();
pendingPlatformSendDispatch = undefined;
};
const remainingFinalContent = (): TelegramDraftMessageSnapshot | undefined => {
@@ -1049,7 +1060,7 @@ export function createTelegramDraftStream(params: {
params.log?.(`telegram stream preview ready (maxChars=${maxChars}, throttleMs=${throttleMs})`);
return {
update: requestDraftUpdate,
update: (text, options) => requestDraftUpdate(text, undefined, options?.onPlatformSendDispatch),
updateLazy: requestLazyDraftUpdate,
updatePreview,
flush,
@@ -11,8 +11,8 @@ import {
buildTtsSupplementMediaPayload,
getReplyPayloadTtsSupplement,
resolveSendableOutboundReplyParts,
type ReplyPayload,
} from "openclaw/plugin-sdk/reply-payload";
import type { ReplyPayload } from "openclaw/plugin-sdk/reply-runtime";
import type { TelegramInlineButtons } from "./button-types.js";
import type { TelegramDraftStream } from "./draft-stream.js";
import type { TelegramPromptContextProjectionSequence } from "./prompt-context-projection.js";
@@ -55,6 +55,8 @@ type CreateLaneTextDelivererParams = {
durable?: boolean;
promptContextSequence?: TelegramPromptContextProjectionSequence;
textMode?: "html";
onPlatformSendDispatch?: () => Promise<void>;
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
},
) => Promise<boolean>;
flushDraftLane: (lane: DraftLaneState) => Promise<void>;
@@ -86,6 +88,8 @@ type DeliverLaneTextParams = {
durable?: boolean;
allowStream?: boolean;
promptContextSequence?: TelegramPromptContextProjectionSequence;
onPlatformSendDispatch?: () => Promise<void>;
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
};
function result(
@@ -276,6 +280,7 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
promptContextSequence: TelegramPromptContextProjectionSequence,
followedByDurablePayload = false,
allowErrorPayload = false,
onPlatformSendDispatch?: () => Promise<void>,
): Promise<LaneDeliveryResult | undefined> => {
const stream = lane.stream;
if (!stream || text.length === 0 || (payload.isError && !allowErrorPayload)) {
@@ -301,8 +306,15 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
lane.lastPartialText = previewText;
lane.hasStreamedMessage = true;
lane.finalized = false;
if (stream.lastDeliveredText?.() !== previewText) {
stream.update(previewText);
const previewAlreadyVisible = stream.lastDeliveredText?.() === previewText;
if (!previewAlreadyVisible) {
if (finalizePreview && onPlatformSendDispatch) {
stream.update(previewText, { onPlatformSendDispatch });
} else {
stream.update(previewText);
}
} else if (finalizePreview) {
await onPlatformSendDispatch?.();
}
if (finalizePreview) {
await params.stopDraftLane(lane);
@@ -341,6 +353,7 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
let buttonsAttached = false;
if (buttons && activeSnapshot) {
try {
await onPlatformSendDispatch?.();
await params.editStreamMessage({
laneName,
messageId,
@@ -383,6 +396,8 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
durable: requestedDurable,
allowStream = true,
promptContextSequence: suppliedPromptContextSequence,
onPlatformSendDispatch,
bindPendingFinalDelivery,
}: DeliverLaneTextParams): Promise<LaneDeliveryResult> => {
const lane = params.lanes[laneName];
const promptContextSequence =
@@ -425,6 +440,7 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
promptContextSequence,
false,
streamedErrorDraftText !== undefined,
onPlatformSendDispatch,
)
: undefined;
if (streamed) {
@@ -449,6 +465,8 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
buttons,
promptContextSequence,
true,
false,
onPlatformSendDispatch,
);
if (finalizedPreview) {
const stripButtons =
@@ -465,6 +483,8 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
afterAcceptedDraft: true,
durable,
promptContextSequence,
onPlatformSendDispatch,
bindPendingFinalDelivery,
},
);
return finalizedPreview;
@@ -491,6 +511,8 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
afterAcceptedDraft,
durable,
promptContextSequence,
onPlatformSendDispatch,
bindPendingFinalDelivery,
...(retainedFinalContent?.sourceTextMode === "html" ? { textMode: "html" } : {}),
},
);
@@ -184,6 +184,35 @@ describe("createLaneTextDeliverer", () => {
expect(harness.lanes.answer.finalized).toBe(true);
});
it("claims an equal visible preview before finalization can crash", async () => {
const events: string[] = [];
const answer = createTestDraftStream({ messageId: 999 });
answer.lastDeliveredText.mockReturnValue(HELLO_FINAL);
const harness = createHarness({ answerStream: answer });
const finalizationCrash = new Error("injected finalization crash");
harness.stopDraftLane.mockImplementationOnce(async () => {
events.push("finalize");
throw finalizationCrash;
});
const onPlatformSendDispatch = vi.fn(async () => {
events.push("custody");
});
await expect(
harness.deliverLaneText({
laneName: "answer",
text: HELLO_FINAL,
payload: { text: HELLO_FINAL },
infoKind: "final",
onPlatformSendDispatch,
}),
).rejects.toBe(finalizationCrash);
expect(events).toEqual(["custody", "finalize"]);
expect(answer.update).not.toHaveBeenCalled();
expect(onPlatformSendDispatch).toHaveBeenCalledOnce();
});
it("streams block and final text through the same lane", async () => {
const harness = createHarness({ answerMessageId: 999 });
@@ -77,6 +77,7 @@ async function resolveTelegramSendContext(params: {
onDeliveryResult?: Parameters<
NonNullable<ChannelOutboundAdapter["sendText"]>
>[0]["onDeliveryResult"];
onPlatformSendDispatch?: () => Promise<void>;
resolveSend: ResolveTelegramSendFn;
}): Promise<{
send: TelegramSendFn;
@@ -93,6 +94,7 @@ async function resolveTelegramSendContext(params: {
silent?: boolean;
gatewayClientScopes?: readonly string[];
onDeliveryResult?: TelegramSendOpts["onDeliveryResult"];
onPlatformSendDispatch?: TelegramSendOpts["onPlatformSendDispatch"];
};
}> {
const send = await params.resolveSend(params.deps);
@@ -113,6 +115,7 @@ async function resolveTelegramSendContext(params: {
await params.onDeliveryResult?.(attachChannelToResult("telegram", result));
}
: undefined,
onPlatformSendDispatch: params.onPlatformSendDispatch,
...(params.formatting?.parseMode === "HTML" ? { textMode: "html" as const } : {}),
tableMode: params.formatting?.tableMode,
},
@@ -383,6 +386,7 @@ export async function sendTelegramPayloadMessages(params: {
if (typeof replyToMessageId !== "number") {
throw new Error("Telegram reaction requires a reply target");
}
await params.baseOpts.onPlatformSendDispatch?.();
const reactionResult = await params.react(params.to, replyToMessageId, reactionEmoji, {
cfg: params.baseOpts.cfg,
accountId: params.baseOpts.accountId,
@@ -597,6 +601,7 @@ export function createTelegramOutboundAdapter(
silent,
isAnonymous,
gatewayClientScopes,
onPlatformSendDispatch,
}) => {
const outboundTo = normalizeTelegramOutboundTarget(to);
const { sendPollTelegram } = await loadSendModule();
@@ -607,6 +612,7 @@ export function createTelegramOutboundAdapter(
silent: silent ?? undefined,
isAnonymous: isAnonymous ?? undefined,
gatewayClientScopes,
onPlatformSendDispatch,
});
},
};
+5 -3
View File
@@ -72,8 +72,9 @@ async function sendLocationTelegramWithContext(
const delivery = await withTelegramNativeQuoteFallback({
label,
requestParams: commonParams,
request: (effectiveParams, retryLabel) =>
prepared.request(
request: async (effectiveParams, retryLabel) => {
await opts.onPlatformSendDispatch?.();
return await prepared.request(
() =>
hasName
? api.sendVenue(
@@ -91,7 +92,8 @@ async function sendLocationTelegramWithContext(
: {}),
} as TelegramSendLocationParams),
retryLabel,
),
);
},
});
const result = delivery.result;
const acceptedParams = toAcceptedThreadScopedParams(delivery.acceptedParams);
+5 -3
View File
@@ -149,14 +149,16 @@ export function createTelegramTextSender(config: {
withTelegramNativeQuoteFallback({
label,
requestParams,
request: (effectiveParams, retryLabel) =>
requestWithChatNotFound(
request: async (effectiveParams, retryLabel) => {
await opts.onPlatformSendDispatch?.();
return await requestWithChatNotFound(
() =>
Object.keys(effectiveParams).length > 0
? api.sendMessage(chatId, messageText, effectiveParams)
: api.sendMessage(chatId, messageText),
retryLabel,
),
);
},
});
const requestPlain = (label: string) =>
requestSendMessage(label, chunk.plainText, plainParams ?? {});
@@ -49,6 +49,8 @@ export type TelegramSendOpts = {
forceDocument?: boolean;
/** Persist each concrete platform send before any later chunk can fail. */
onDeliveryResult?: (result: TelegramSendResult) => Promise<void> | void;
/** @internal Refresh durable custody immediately before Telegram Bot API I/O. */
onPlatformSendDispatch?: () => Promise<void>;
};
export type TelegramApiCallOpts = Pick<
@@ -76,5 +78,10 @@ export type TelegramSendResult = {
export type TelegramLocationSendOpts = TelegramThreadedSendOpts &
Pick<
TelegramSendOpts,
"buttons" | "quoteText" | "promptContextProjectionPlan" | "silent" | "onDeliveryResult"
| "buttons"
| "quoteText"
| "promptContextProjectionPlan"
| "silent"
| "onDeliveryResult"
| "onPlatformSendDispatch"
>;
+5 -3
View File
@@ -288,12 +288,14 @@ async function sendMessageTelegramWithContext(
withTelegramNativeQuoteFallback({
label,
requestParams,
request: (effectiveParams, effectiveLabel) =>
requestWithChatNotFound(
request: async (effectiveParams, effectiveLabel) => {
await opts.onPlatformSendDispatch?.();
return await requestWithChatNotFound(
() => sender(effectiveParams),
effectiveLabel,
shouldLog ? { shouldLog } : undefined,
),
);
},
}),
});
};
+2 -1
View File
@@ -102,7 +102,7 @@ async function sendStickerTelegramWithContext(
}
type TelegramPollOpts = TelegramThreadedSendOpts &
Pick<TelegramSendOpts, "silent"> & {
Pick<TelegramSendOpts, "onPlatformSendDispatch" | "silent"> & {
/** Whether votes are anonymous. Defaults to true (Telegram default). */
isAnonymous?: boolean;
};
@@ -163,6 +163,7 @@ async function sendPollTelegramWithContext(
...(opts.silent === true ? { disable_notification: true } : {}),
};
await opts.onPlatformSendDispatch?.();
const result = await prepared.request(
() =>
api.sendPoll(prepared.chatId, normalizedPoll.question, normalizedPoll.options, pollParams),
@@ -828,7 +828,7 @@ describe("agentCommand compaction transcript rotation", () => {
},
);
it("skips post-turn compaction before delivering sendable finals that pending text cannot replay", async () => {
it("compacts after persisting transport ownership for finals that text cannot replay", async () => {
const sessionId = "unrecoverable-media-before-compaction";
const sessionKey = `agent:main:explicit:${sessionId}`;
const payloads = [{ mediaUrl: "/tmp/reply.ogg", audioAsVoice: true }];
@@ -845,7 +845,7 @@ describe("agentCommand compaction transcript rotation", () => {
deliver: true,
});
expect(state.runCliTurnCompactionLifecycleMock).not.toHaveBeenCalled();
expect(state.runCliTurnCompactionLifecycleMock).toHaveBeenCalledOnce();
expect(state.deliverAgentCommandResultMock).toHaveBeenCalledOnce();
expect(state.deliverAgentCommandResultMock).toHaveBeenCalledWith(
expect.objectContaining({ payloads }),
+47 -4
View File
@@ -1,5 +1,6 @@
import { getReplyPayloadMetadata } from "../../auto-reply/reply-payload.js";
import type { CliDeps } from "../../cli/deps.types.js";
import { buildRestartRecoveryClaimCleanupPatch } from "../../config/sessions/restart-recovery-state.js";
import type { RestartRecoveryTerminalDeliveryEvidenceResult } from "../../config/sessions/restart-recovery-types.js";
import type { SessionEntry } from "../../config/sessions/types.js";
import { assertAgentRunLifecycleGenerationCurrent } from "../../infra/agent-events.js";
@@ -70,6 +71,7 @@ export async function finalizeEmbeddedAgentCommand(params: {
cwd,
agentDir,
outboundSession,
runId,
agentCfg,
} = params.prepared;
const {
@@ -374,7 +376,8 @@ export async function finalizeEmbeddedAgentCommand(params: {
!params.suppressVisibleSessionEffects &&
!sessionReboundDuringRun
) {
const entry = sessionStore[sessionKey] ?? sessionEntry;
const entry =
(await resolveFreshSessionEntryForDelivery?.()) ?? sessionStore[sessionKey] ?? sessionEntry;
if (!entry) {
throw new Error("Cannot clear pending delivery without a session entry");
}
@@ -383,15 +386,55 @@ export async function finalizeEmbeddedAgentCommand(params: {
params.opts.deliver === true &&
!pendingFinalDeliveryMarker.hasSendableFinalPayload &&
entry.pendingFinalDelivery?.kind === "transport-only";
if (deliveryResult?.deliverySucceeded === true || clearStaleTransportOnly) {
const clearOwnedPendingFinal =
deliveryResult?.deliverySucceeded === true &&
pendingFinalDeliveryMarker.pendingFinalDeliveryIntentId !== undefined;
// Preserve the exact local claim through sibling session writes so a delivered
// source is tombstoned before admission release can erase its ownership fields.
const recoveryClaimEntry =
entry.restartRecoveryDeliveryRunId === runId
? entry
: sessionEntry?.restartRecoveryDeliveryRunId === runId
? sessionEntry
: params.sessionEntry?.restartRecoveryDeliveryRunId === runId
? params.sessionEntry
: undefined;
if (clearOwnedPendingFinal || clearStaleTransportOnly || recoveryClaimEntry) {
const now = Date.now();
sessionEntry = await persistAgentSession({
sessionStore,
sessionKey,
storePath,
initialEntry: entry,
entry: clearPendingFinalDelivery(entry, Date.now()),
entry: {
...(clearOwnedPendingFinal || clearStaleTransportOnly
? clearPendingFinalDelivery(entry, now)
: { ...entry, updatedAt: now }),
...(recoveryClaimEntry
? buildRestartRecoveryClaimCleanupPatch({
entry: {
...recoveryClaimEntry,
restartRecoveryTerminalDeliveryEvidence:
entry.restartRecoveryTerminalDeliveryEvidence,
restartRecoveryTerminalRunIds: entry.restartRecoveryTerminalRunIds,
},
recordTerminalSource: true,
terminalDeliveryEvidence: buildRestartRecoveryTerminalDeliveryEvidence(
deliveryResult ?? result,
),
terminalRunId: runId,
})
: {}),
},
shouldPersist: (current) =>
shouldPersistCurrentRunSessionCleanup(current, runOwnedSessionId),
shouldPersistCurrentRunSessionCleanup(current, runOwnedSessionId) &&
(!recoveryClaimEntry ||
current?.restartRecoveryDeliveryRunId === undefined ||
current.restartRecoveryDeliveryRunId === runId) &&
(!clearOwnedPendingFinal ||
current?.pendingFinalDelivery?.intentId ===
pendingFinalDeliveryMarker.pendingFinalDeliveryIntentId) &&
(!clearStaleTransportOnly || current?.pendingFinalDelivery?.kind === "transport-only"),
});
}
}
@@ -14,6 +14,7 @@ export function scheduleMainSessionRecoveryPendingTarget(
getConfig: getRuntimeConfig,
getGatewayRuntime: getGatewayRecoveryRuntime,
sessionKey: target.sessionKey,
stateDir: target.stateDir,
storePath: target.storePath,
}),
() => {}, // Startup recovery remains the fallback if this optional module cannot load.
@@ -32,6 +32,7 @@ type MainSessionRecoveryStoreResult = {
export type MainSessionRecoveryPendingTarget = MainSessionRecoveryStoreTarget & {
sessionId: string;
stateDir?: string;
};
function matchesReservation(entry: SessionEntry, reservation: MainSessionRecoveryReservation) {
@@ -298,6 +298,7 @@ export async function markSessionCompletedAfterRecoveryCheckpoint(params: {
agentId: string;
entry: SessionEntry;
messages: readonly unknown[];
pendingFinalDeliveryIntentId?: string;
reason: "delivered-terminal" | "delivered-terminal-receipt" | "handled-silent";
storePath: string;
sessionKey: string;
@@ -321,6 +322,7 @@ export async function markSessionCompletedAfterRecoveryCheckpoint(params: {
pendingFinalDelivery: undefined,
restartRecoveryForceSafeTools: undefined,
restartRecoveryRuns: undefined,
...buildMainSessionRecoveryClearPatch(params.entry),
runtimeMs:
typeof params.entry.startedAt === "number"
? Math.max(0, endedAt - params.entry.startedAt)
@@ -471,6 +473,8 @@ export async function markSessionCompletedAfterRecoveryCheckpoint(params: {
if (
!entry ||
entry.sessionId !== params.entry.sessionId ||
(params.pendingFinalDeliveryIntentId !== undefined &&
entry.pendingFinalDelivery?.intentId !== params.pendingFinalDeliveryIntentId) ||
entry.status !== "running" ||
entry.abortedLastRun !== true ||
normalizeOptionalString(entry.restartRecoveryDeliveryRunId) !== expectedRecoveryRunId ||
@@ -86,6 +86,7 @@ export async function failUnresumableMainSession(params: {
gatewayRuntime: GatewayRecoveryRuntime;
observation: MainSessionRecoveryObservation;
reason: string;
noticeText?: string;
sessionKey: string;
storePath: string;
}): Promise<"failed" | "skipped"> {
@@ -105,6 +106,7 @@ export async function failUnresumableMainSession(params: {
entry: params.entry,
sessionKey: params.sessionKey,
storePath: params.storePath,
...(params.noticeText ? { text: params.noticeText } : {}),
})) !== "written"
) {
// Keep ownership for another recovery attempt until its terminal notice is durable.
@@ -126,7 +128,7 @@ export async function failUnresumableMainSession(params: {
gatewayRuntime: params.gatewayRuntime,
reason: params.reason,
sessionKey: params.sessionKey,
text: UNRESUMABLE_SESSION_NOTICE,
text: params.noticeText ?? UNRESUMABLE_SESSION_NOTICE,
});
}
return "failed";
@@ -88,6 +88,7 @@ export async function recoverRestartAbortedMainSessions(params: {
cfg: params.cfg,
onExhaustedTarget: params.onExhaustedTarget,
storePath,
stateDir: params.stateDir,
resumedSessionKeys,
activeSessionIds: params.activeSessionIds,
activeSessionKeys: params.activeSessionKeys,
@@ -116,6 +117,7 @@ export async function retryRestartAbortedMainSessionRecovery(params: {
expectedRecoverySourceRunId?: string;
expectedSessionId: string;
sessionKey: string;
stateDir?: string;
storePath: string;
gatewayRuntime: GatewayRecoveryRuntime;
}): Promise<RecoveryCounts> {
@@ -147,6 +149,7 @@ async function recoverExpectedRestartRecovery(params: {
sessionKey: string;
shouldContinue?: () => boolean;
storePath: string;
stateDir?: string;
gatewayRuntime: GatewayRecoveryRuntime;
}): Promise<RecoveryCounts> {
const loadExpected = () =>
@@ -186,6 +189,7 @@ async function recoverExpectedRestartRecovery(params: {
cfg: params.cfg,
observationOnly: params.observationOnly,
storePath: params.storePath,
stateDir: params.stateDir,
resumedSessionKeys: new Set<string>(),
expectedClaim: params.expectedClaim,
expectedTarget: params.expectedTarget,
@@ -208,6 +212,7 @@ export function scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease(param
maxRetries?: number;
expectedSessionId: string;
sessionKey: string;
stateDir?: string;
storePath: string;
}): void {
const recover = () =>
@@ -220,6 +225,7 @@ export function scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease(param
cfg: params.getConfig(),
expectedSessionId: params.expectedSessionId,
sessionKey: params.sessionKey,
stateDir: params.stateDir,
storePath: params.storePath,
gatewayRuntime,
});
@@ -327,6 +333,7 @@ export function scheduleRestartAbortedMainSessionRecovery(params: {
sessionKey: target.sessionKey,
shouldContinue,
storePath: target.storePath,
stateDir: params.stateDir,
gatewayRuntime: params.gatewayRuntime,
}),
),
@@ -14,6 +14,7 @@ import type { GatewayRecoveryRuntime } from "../../gateway/server-instance-runti
import { readSessionMessagesAsync } from "../../gateway/session-transcript-readers.js";
import { resolveGatewaySessionStoreTarget } from "../../gateway/session-utils.js";
import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
import { findDeliveryIntentOwner } from "../../infra/outbound/delivery-queue-storage.js";
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
import { resolveDefaultAgentId } from "../agent-scope-config.js";
import {
@@ -53,6 +54,32 @@ import {
normalizeStringSet,
} from "./main-session-restart-recovery-shared.js";
function pendingFinalRecoveryAction(
pending: NonNullable<SessionEntry["pendingFinalDelivery"]>,
stateDir?: string,
): "complete" | "defer" | "fail" | "retry" {
const deliveries = pending.deliveries;
if (!deliveries?.length) {
return "fail";
}
if (deliveries.every(({ state }) => state === "delivered" || state === "suppressed")) {
return "complete";
}
const owners = deliveries.map(({ id }) => findDeliveryIntentOwner(id, stateDir));
if (owners.some((owner) => owner?.status === "pending")) {
return "defer";
}
for (const [index, delivery] of deliveries.entries()) {
const owner = owners[index];
if (owner || delivery.state === "delivered" || delivery.state === "unknown") {
return "fail";
}
}
return pending.kind === "replayable" && deliveries.every(({ state }) => state === "prepared")
? "retry"
: "fail";
}
export function loadExpectedRestartRecoveryTarget(params: {
expected: ExpectedRestartRecoveryTarget;
storePath: string;
@@ -99,6 +126,7 @@ export async function recoverStore(params: {
observationOnly?: boolean;
onExhaustedTarget?: (target: ExhaustedRestartRecoveryTarget) => void;
storePath: string;
stateDir?: string;
resumedSessionKeys: Set<string>;
expectedClaim?: ExpectedRestartRecoveryClaim;
expectedTarget?: ExpectedRestartRecoveryTarget;
@@ -293,7 +321,7 @@ export async function recoverStore(params: {
}
}
};
const failCurrent = async (reason: string) => {
const failCurrent = async (reason: string, noticeText?: string) => {
if (stopped()) {
return false;
}
@@ -303,6 +331,7 @@ export async function recoverStore(params: {
gatewayRuntime: params.gatewayRuntime,
observation: recoveryView.observation,
reason,
...(noticeText ? { noticeText } : {}),
sessionKey,
storePath: params.storePath,
});
@@ -363,6 +392,44 @@ export async function recoverStore(params: {
);
};
const pendingAction = entry.pendingFinalDelivery
? pendingFinalRecoveryAction(entry.pendingFinalDelivery, params.stateDir)
: undefined;
if (pendingAction === "defer") {
result.failed++;
continue;
}
if (pendingAction === "complete") {
const completion = await markSessionCompletedAfterRecoveryCheckpoint({
agentId,
entry,
messages: [],
pendingFinalDeliveryIntentId: entry.pendingFinalDelivery?.intentId,
reason: "delivered-terminal-receipt",
sessionKey,
storePath: params.storePath,
});
if (completion.outcome === "completed") {
params.resumedSessionKeys.add(resumeDedupeKey);
result.recovered++;
} else {
result.skipped++;
}
continue;
}
if (pendingAction === "fail") {
if (
!(await failCurrent(
"pending final delivery outcome is unknown",
"My previous response was interrupted during delivery. " +
"Please ask for any missing remainder; I won't rerun your previous request automatically.",
))
) {
return result;
}
continue;
}
if (
entry.pendingFinalDelivery?.kind === "replayable" &&
entry.restartRecoveryForceSafeTools === true
@@ -26,6 +26,9 @@ import {
rotateAgentEventLifecycleGeneration,
} from "../../infra/agent-events.js";
import { registerAgentRunContext } from "../../infra/agent-run-registry.js";
import { moveDeliveryQueueEntryToFailed } from "../../infra/delivery-queue-sqlite.js";
import { OUTBOUND_DELIVERY_QUEUE_NAME } from "../../infra/outbound/delivery-queue-media-staging.js";
import { ackDelivery, enqueueDeliveryOnce } from "../../infra/outbound/delivery-queue-storage.js";
import {
initializeGlobalHookRunner,
resetGlobalHookRunner,
@@ -262,6 +265,8 @@ function makePendingFinalDelivery(
kind: "replayable",
text,
createdAt: Date.now(),
intentId: "intent-prepared-default",
deliveries: [{ id: "delivery-prepared-default", state: "prepared" }],
...overrides,
};
}
@@ -1833,65 +1838,290 @@ describe("main-session-restart-recovery", () => {
expect(store["agent:main:main"]?.abortedLastRun).toBe(true);
});
it("resumes marked sessions with a durable pending final delivery payload (Phase 2)", async () => {
it.each([
["missing", undefined],
["empty", []],
] as const)(
"fails closed when pending final delivery identities are %s",
async (_, deliveries) => {
const sessionsDir = await makeSessionsDir();
const pendingPayload = "The final answer is 42.";
await writeMainSession({
sessionsDir,
restartRecoveryForceSafeTools: true,
pendingFinalDelivery: {
kind: "replayable",
text: pendingPayload,
createdAt: Date.now() - 5_000,
...(deliveries ? { deliveries: [...deliveries] } : {}),
context: {
channel: "discord",
to: "discord:dm:final",
accountId: "main",
},
},
restartRecoveryBeforeAgentReplyState: "handled-reply",
restartRecoveryDeliveryRunId: "discord-message-1",
restartRecoveryDeliverySourceRunId: "discord-message-1",
restartRecoverySourceIngress: "channel",
restartRecoveryDeliveryContext: {
channel: "discord",
to: "discord:dm:stale",
accountId: "old",
},
});
await writeTranscript(sessionsDir, "main-session", [
{ role: "user", content: "calculate the answer" },
{ role: "assistant", content: [{ type: "toolCall", id: "call-1", name: "calc" }] },
{ role: "toolResult", content: "42" },
]);
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 }, {});
expect(runtimePluginMocks.findRestartRecoveryUnsafeReplyHook).not.toHaveBeenCalled();
expect(callGateway).not.toHaveBeenCalled();
expect(sendRecoveryNotice).toHaveBeenCalledWith(
expect.objectContaining({ text: expect.stringContaining("ask for any missing remainder") }),
);
},
);
it("retries a prepared pending final only when no queue owner exists", async () => {
const sessionsDir = await makeSessionsDir();
const pendingPayload = "The final answer is 42.";
await writeMainSession({
sessionsDir,
restartRecoveryForceSafeTools: true,
pendingFinalDelivery: {
kind: "replayable",
text: pendingPayload,
createdAt: Date.now() - 5_000,
context: {
channel: "discord",
to: "discord:dm:final",
accountId: "main",
},
},
restartRecoveryBeforeAgentReplyState: "handled-reply",
restartRecoveryDeliveryRunId: "discord-message-1",
restartRecoveryDeliverySourceRunId: "discord-message-1",
restartRecoverySourceIngress: "channel",
restartRecoveryDeliveryContext: {
channel: "discord",
to: "discord:dm:stale",
accountId: "old",
},
pendingFinalDelivery: makePendingFinalDelivery("The prepared final answer.", {
intentId: "intent-prepared",
deliveries: [{ id: "delivery-prepared", state: "prepared" }],
}),
});
await writeTranscript(sessionsDir, "main-session", [
{ role: "user", content: "calculate the answer" },
{ role: "assistant", content: [{ type: "toolCall", id: "call-1", name: "calc" }] },
{ role: "toolResult", content: "42" },
{ role: "user", content: "finish the answer" },
]);
await expectRecovery({ recovered: 1, failed: 0, skipped: 0 }, {});
expect(runtimePluginMocks.findRestartRecoveryUnsafeReplyHook).toHaveBeenCalledWith({
trigger: "user",
});
expect(callGateway).toHaveBeenCalledOnce();
expect(gatewayParams()).toMatchObject({
deliver: true,
bestEffortDeliver: true,
channel: "discord",
to: "discord:dm:final",
accountId: "main",
forceRestartSafeTools: true,
});
expect(gatewayParams().message).toContain(pendingPayload);
await expectRecovery({ recovered: 1, failed: 0, skipped: 0 });
const beforeStoreRead = Date.now();
const store = readStore(path.join(sessionsDir, "sessions.json"));
const entry = store["agent:main:main"];
expect(entry?.abortedLastRun).toBe(false);
expect(entry?.pendingFinalDelivery).toMatchObject({
kind: "replayable",
text: pendingPayload,
});
expect(entry?.restartRecoveryForceSafeTools).toBe(true);
expect(entry?.pendingFinalDelivery?.createdAt).toBeLessThanOrEqual(beforeStoreRead);
expect(callGateway).toHaveBeenCalledOnce();
expect(gatewayParams().message).toContain("The prepared final answer.");
});
it("quietly completes a pending final whose deliveries are terminal", async () => {
const sessionsDir = await makeSessionsDir();
const storePath = path.join(sessionsDir, "sessions.json");
await writeMainSession({
sessionsDir,
pendingFinalDelivery: makePendingFinalDelivery("Already delivered.", {
intentId: "intent-delivered",
deliveries: [
{ id: "delivery-delivered", state: "delivered" },
{ id: "delivery-suppressed", state: "suppressed" },
],
}),
});
await expectRecovery({ recovered: 1, failed: 0, skipped: 0 });
expect(callGateway).not.toHaveBeenCalled();
expect(sendRecoveryNotice).not.toHaveBeenCalled();
expect(loadSessionEntry({ sessionKey: "agent:main:main", storePath })).toMatchObject({
status: "done",
abortedLastRun: false,
});
expect(
loadSessionEntry({ sessionKey: "agent:main:main", storePath })?.pendingFinalDelivery,
).toBeUndefined();
});
it.each([
[
{ id: "delivery-already-delivered", state: "delivered" as const },
{ id: "delivery-still-pending", state: "queued" as const },
],
[
{ id: "delivery-still-pending", state: "queued" as const },
{ id: "delivery-already-delivered", state: "delivered" as const },
],
])("defers mixed deliveries while any exact queue owner is pending", async (...deliveries) => {
try {
await enqueueDeliveryOnce(
{
channel: "discord",
to: "discord:dm:123",
payloads: [{ text: "Pending sibling." }],
queuePolicy: "required",
completionRetention: "permanent",
},
"delivery-still-pending",
tmpDir,
);
const sessionsDir = await makeSessionsDir();
await writeMainSession({
sessionsDir,
pendingFinalDelivery: makePendingFinalDelivery("Partially delivered answer.", {
context: discordDeliveryContext,
intentId: "intent-mixed-pending",
deliveries,
}),
});
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
expect(callGateway).not.toHaveBeenCalled();
expect(sendRecoveryNotice).not.toHaveBeenCalled();
} finally {
closeOpenClawStateDatabaseForTest();
}
});
it("completes terminal deliveries despite a residual pending queue row", async () => {
try {
await enqueueDeliveryOnce(
{
channel: "discord",
to: "discord:dm:123",
payloads: [{ text: "Already delivered." }],
queuePolicy: "required",
completionRetention: "permanent",
},
"delivery-terminal-with-row",
tmpDir,
);
const sessionsDir = await makeSessionsDir();
const storePath = path.join(sessionsDir, "sessions.json");
await writeMainSession({
sessionsDir,
pendingFinalDelivery: makePendingFinalDelivery("Already delivered.", {
intentId: "intent-terminal-with-row",
deliveries: [{ id: "delivery-terminal-with-row", state: "delivered" }],
}),
});
await expectRecovery({ recovered: 1, failed: 0, skipped: 0 });
expect(callGateway).not.toHaveBeenCalled();
expect(sendRecoveryNotice).not.toHaveBeenCalled();
expect(loadSessionEntry({ sessionKey: "agent:main:main", storePath })?.status).toBe("done");
} finally {
closeOpenClawStateDatabaseForTest();
}
});
it("fails closed for an unqueued media-only final", async () => {
const sessionsDir = await makeSessionsDir();
await writeMainSession({
sessionsDir,
pendingFinalDelivery: {
kind: "transport-only",
createdAt: Date.now(),
intentId: "intent-media-only",
deliveries: [{ id: "delivery-media-only", state: "prepared" }],
context: discordDeliveryContext,
},
});
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
expect(callGateway).not.toHaveBeenCalled();
expect(sendRecoveryNotice).toHaveBeenCalledWith(
expect.objectContaining({ text: expect.stringContaining("ask for any missing remainder") }),
);
});
it("fails visibly instead of replaying part of an unqueued text and media final", async () => {
const sessionsDir = await makeSessionsDir();
await writeMainSession({
sessionsDir,
pendingFinalDelivery: {
kind: "transport-only",
createdAt: Date.now(),
context: discordDeliveryContext,
intentId: "intent-text-media",
deliveries: [
{ id: "delivery-text", state: "prepared" },
{ id: "delivery-media", state: "prepared" },
],
},
});
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
expect(callGateway).not.toHaveBeenCalled();
expect(sendRecoveryNotice).toHaveBeenCalledOnce();
});
it.each(["delivered", "unknown"] as const)(
"fails closed when a %s delivery is mixed with prepared work",
async (state) => {
const sessionsDir = await makeSessionsDir();
await writeMainSession({
sessionsDir,
pendingFinalDelivery: makePendingFinalDelivery("Do not regenerate this aggregate.", {
context: discordDeliveryContext,
intentId: `intent-mixed-${state}`,
deliveries: [
{ id: `delivery-${state}`, state },
{ id: "delivery-still-prepared", state: "prepared" },
],
}),
});
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
expect(callGateway).not.toHaveBeenCalled();
expect(sendRecoveryNotice).toHaveBeenCalledOnce();
expect(sendRecoveryNotice).toHaveBeenCalledWith(
expect.objectContaining({
text: expect.stringContaining("ask for any missing remainder"),
}),
);
expect(sendRecoveryNotice.mock.calls[0]?.[0].text).not.toContain("send that last request");
},
);
it.each(["pending", "failed", "completed"] as const)(
"does not regenerate a prepared pending final while its exact queue owner is %s",
async (ownerStatus) => {
const deliveryId = `delivery-owner-${ownerStatus}`;
try {
await enqueueDeliveryOnce(
{
channel: "discord",
to: "discord:dm:123",
payloads: [{ text: "Queue owns this final." }],
queuePolicy: "required",
completionRetention: "permanent",
},
deliveryId,
tmpDir,
);
if (ownerStatus === "failed") {
moveDeliveryQueueEntryToFailed(OUTBOUND_DELIVERY_QUEUE_NAME, deliveryId, tmpDir);
} else if (ownerStatus === "completed") {
await ackDelivery(deliveryId, tmpDir);
}
const sessionsDir = await makeSessionsDir();
await writeMainSession({
sessionsDir,
pendingFinalDelivery: makePendingFinalDelivery("Queue owns this final.", {
context: discordDeliveryContext,
intentId: `intent-owner-${ownerStatus}`,
deliveries: [{ id: deliveryId, state: "prepared" }],
}),
});
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
expect(callGateway).not.toHaveBeenCalled();
if (ownerStatus === "pending") {
expect(sendRecoveryNotice).not.toHaveBeenCalled();
} else {
expect(sendRecoveryNotice).toHaveBeenCalledOnce();
}
} finally {
closeOpenClawStateDatabaseForTest();
}
},
);
it("keeps a hook-owned pending final behind the unsafe-hook gate after claim cleanup", async () => {
const sessionsDir = await makeSessionsDir();
const storePath = path.join(sessionsDir, "sessions.json");
@@ -1964,11 +2194,9 @@ describe("main-session-restart-recovery", () => {
].join("\n");
await writeMainSession({
sessionsDir,
pendingFinalDelivery: {
kind: "replayable",
text: pendingPayload,
pendingFinalDelivery: makePendingFinalDelivery(pendingPayload, {
createdAt: Date.now() - 5_000,
},
}),
});
await writeTranscript(sessionsDir, "main-session", [
{ role: "user", content: "calculate the answer" },
@@ -0,0 +1,46 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { getReplyPayloadMetadata } from "../auto-reply/reply-payload.js";
import type { SessionEntry } from "../config/sessions/types.js";
import { persistPendingFinalDeliveryMarker } from "./pending-final-delivery-marker.js";
const state = vi.hoisted(() => ({ persistAgentSession: vi.fn() }));
vi.mock("./command/attempt-execution.shared.js", () => ({
persistAgentSession: (...args: unknown[]) => state.persistAgentSession(...args),
}));
describe("persistPendingFinalDeliveryMarker", () => {
beforeEach(() => {
state.persistAgentSession
.mockReset()
.mockImplementation(async (params: { entry: SessionEntry }) => params.entry);
});
it("owns a multi-payload command delivery as one durable batch", async () => {
const entry: SessionEntry = { sessionId: "session-1", updatedAt: 1 };
const payloads = [{ text: "first" }, { text: "second" }];
const result = await persistPendingFinalDeliveryMarker({
deliver: true,
sessionStore: { main: entry },
sessionKey: "main",
sessionEntry: entry,
storePath: "/tmp/sessions.json",
suppressVisibleSessionEffects: false,
sessionReboundDuringRun: false,
payloads,
deliveryContext: { channel: "discord", to: "channel:c1" },
runOwnedSessionId: "session-1",
});
expect(result.sessionEntry?.pendingFinalDelivery?.deliveries).toEqual([
{ id: expect.any(String), state: "prepared" },
]);
const deliveryId = result.sessionEntry?.pendingFinalDelivery?.deliveries?.[0]?.id;
expect(
payloads.map(
(payload) => getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion?.deliveryId,
),
).toEqual([deliveryId, deliveryId]);
});
});
+38 -11
View File
@@ -1,5 +1,6 @@
/** Persists restart-recoverable final delivery markers for agent runs. */
import type { ReplyPayload } from "../auto-reply/reply-payload.js";
import { randomUUID } from "node:crypto";
import { setReplyPayloadMetadata, type ReplyPayload } from "../auto-reply/reply-payload.js";
import {
buildRecoverablePendingFinalDeliveryText,
normalizePendingFinalDeliveryPayloads,
@@ -26,15 +27,20 @@ type PersistPendingFinalDeliveryMarkerParams = {
type PendingFinalDeliveryMarkerResult = {
sessionEntry?: SessionEntry;
pendingFinalDeliveryMarkerPersisted: boolean;
pendingFinalDeliveryIntentId?: string;
hasSendableFinalPayload: boolean;
};
export async function persistPendingFinalDeliveryMarker(
params: PersistPendingFinalDeliveryMarkerParams,
): Promise<PendingFinalDeliveryMarkerResult> {
const recoveryPayloads = normalizePendingFinalRecoveryPayloads(params.payloads);
const hasSendableFinalPayload = normalizePendingFinalDeliveryPayloads(params.payloads).length > 0;
const recoverableText = buildRecoverablePendingFinalDeliveryText(recoveryPayloads);
const sendablePayloads = params.payloads.filter(
(payload) => normalizePendingFinalDeliveryPayloads([payload]).length > 0,
);
const hasSendableFinalPayload = sendablePayloads.length > 0;
const recoverableText = buildRecoverablePendingFinalDeliveryText(
normalizePendingFinalRecoveryPayloads(params.payloads),
);
if (
!params.deliver ||
@@ -42,10 +48,10 @@ export async function persistPendingFinalDeliveryMarker(
!params.sessionKey ||
params.suppressVisibleSessionEffects ||
params.sessionReboundDuringRun ||
params.payloads.length === 0 ||
isSubagentSessionKey(params.sessionKey) ||
!recoverableText ||
!hasSendableFinalPayload ||
// A run without a resolvable delivery route (e.g. rejected best-effort
// target) must not leave a custody marker restart recovery could act on.
!params.deliveryContext
) {
return {
@@ -65,6 +71,8 @@ export async function persistPendingFinalDeliveryMarker(
}
const now = Date.now();
const intentId = randomUUID();
const deliveryId = randomUUID();
const persisted = await persistAgentSession({
sessionStore: params.sessionStore,
sessionKey: params.sessionKey,
@@ -73,8 +81,11 @@ export async function persistPendingFinalDeliveryMarker(
entry: {
...entry,
pendingFinalDelivery: {
kind: "replayable",
text: recoverableText,
...(recoverableText
? { kind: "replayable" as const, text: recoverableText }
: { kind: "transport-only" as const }),
intentId,
deliveries: [{ id: deliveryId, state: "prepared" as const }],
createdAt: now,
context: params.deliveryContext,
},
@@ -83,13 +94,29 @@ export async function persistPendingFinalDeliveryMarker(
shouldPersist: (current) =>
current?.sessionId === params.runOwnedSessionId && current.abortedLastRun !== true,
});
const markerPersisted =
persisted?.pendingFinalDelivery?.kind === "replayable" &&
persisted.pendingFinalDelivery.text === recoverableText;
const markerPersisted = persisted?.pendingFinalDelivery?.intentId === intentId;
if (markerPersisted) {
for (const payload of sendablePayloads) {
setReplyPayloadMetadata(payload, {
pendingFinalDeliveryCompletion: {
deliveryId,
intentId,
...(entry.restartRecoveryDeliveryRunId
? { recoveryRunId: entry.restartRecoveryDeliveryRunId }
: {}),
sessionId: params.runOwnedSessionId,
sessionKey: params.sessionKey,
storePath: params.storePath,
},
});
}
}
return {
sessionEntry: persisted,
pendingFinalDeliveryMarkerPersisted: markerPersisted,
...(markerPersisted ? { pendingFinalDeliveryIntentId: intentId } : {}),
hasSendableFinalPayload,
};
}
+9 -4
View File
@@ -245,10 +245,15 @@ export type ReplyPayloadMetadata = {
};
/** Opaque owner for one final-delivery transcript capture on a shared dispatcher. */
finalDeliveryCapture?: object;
/** Durable pending-final intent represented by this runtime payload. */
pendingFinalDeliveryIntentId?: string;
/** Restart-safe text this payload contributes to its pending-final intent. */
pendingFinalDeliveryRetryText?: string;
/** Exact persisted delivery owner; WeakMap-only and never serialized. */
pendingFinalDeliveryCompletion?: {
deliveryId: string;
intentId: string;
recoveryRunId?: string;
sessionId: string;
sessionKey: string;
storePath: string;
};
/** replyToId existed before reply threading could inject an implicit target. */
replyToIdExplicit?: boolean;
/** Canonical reply policy used by both message-tool dedupe and final delivery routing. */
+1 -17
View File
@@ -19,7 +19,6 @@ import {
normalizeDeliveryContext,
} from "../../utils/delivery-context.shared.js";
import { resolveFallbackTransition } from "../fallback-state.js";
import { stripHeartbeatToken } from "../heartbeat.js";
import {
isReplyPayloadStatusNotice,
markReplyPayloadForSourceSuppressionDelivery,
@@ -37,10 +36,7 @@ import type { BlockReplyPipeline } from "./block-reply-pipeline.js";
import { resolveEffectiveReplyRoute } from "./effective-reply-route.js";
import type { InternalGetReplyOptions } from "./get-reply.types.js";
import { normalizeReplyPayload } from "./normalize-reply.js";
import {
buildPendingFinalDeliveryText,
sanitizePendingFinalDeliveryText,
} from "./pending-final-delivery.js";
import { sanitizePendingFinalDeliveryText } from "./pending-final-delivery.js";
import { type FollowupRun, type QueueSettings, scheduleFollowupDrain } from "./queue.js";
import { normalizeReplyPayloadDirectives } from "./reply-delivery.js";
import { type ReplyOperation, runAfterReplyOperationClear } from "./reply-run-registry.js";
@@ -77,18 +73,6 @@ export function markBeforeAgentRunBlockedPayloads(payloads: ReplyPayload[]): Rep
);
}
export function resolvePendingFinalDeliveryRetryText(params: {
isHeartbeat: boolean;
payload: ReplyPayload;
}): string {
const pendingText = buildPendingFinalDeliveryText([params.payload]);
if (!params.isHeartbeat) {
return pendingText;
}
const stripped = stripHeartbeatToken(pendingText, { mode: "message" });
return stripped.shouldSkip ? "" : stripped.text || pendingText;
}
export function buildSilentFallbackFailurePayload(params: {
fallbackTransition: ReturnType<typeof resolveFallbackTransition>;
fallbackFailureKnown: boolean;
+12 -2
View File
@@ -319,15 +319,25 @@ export async function executePreparedReplyAgentRun(
});
if (!sourceReplyPolicy.suppressDelivery) {
const pendingFinalDeliveryIntentId = crypto.randomUUID();
const pendingFinalDeliveryDeliveryId = crypto.randomUUID();
setReplyPayloadMetadata(hookReply, {
pendingFinalDeliveryIntentId,
pendingFinalDeliveryRetryText: hookFinalDeliveryText,
pendingFinalDeliveryCompletion: {
deliveryId: pendingFinalDeliveryDeliveryId,
intentId: pendingFinalDeliveryIntentId,
...(activeSessionEntry?.restartRecoveryDeliveryRunId
? { recoveryRunId: activeSessionEntry.restartRecoveryDeliveryRunId }
: {}),
sessionId: replyOperation.sessionId,
sessionKey,
storePath,
},
});
hookCheckpoint = {
state: hookFinalDeliveryText ? "handled-reply" : "handled-unrecoverable",
pendingFinalDelivery: {
text: hookFinalDeliveryText ?? "",
intentId: pendingFinalDeliveryIntentId,
deliveries: [{ id: pendingFinalDeliveryDeliveryId, state: "prepared" }],
context: resolveReplyRunDeliveryContext({
cfg,
sessionCtx,
@@ -14,7 +14,6 @@ import type { ReplyPayload } from "../types.js";
import {
buildInlinePluginStatusPayload,
markBeforeAgentRunBlockedPayloads,
resolvePendingFinalDeliveryRetryText,
resolveReplyRunDeliveryContext,
resolveSourceReplyPolicy,
} from "./agent-runner-core.js";
@@ -35,7 +34,11 @@ import {
mergeExecutionTrace,
} from "./agent-runner-trace.js";
import { appendUsageLine } from "./agent-runner-usage-line.js";
import { buildPendingFinalDeliveryText } from "./pending-final-delivery.js";
import {
buildRecoverablePendingFinalDeliveryText,
normalizePendingFinalDeliveryPayloads,
normalizePendingFinalRecoveryPayloads,
} from "./pending-final-delivery.js";
import { readPostCompactionContext } from "./post-compaction-context.js";
import { warnPrivateMessageToolFinal } from "./private-message-tool-final.js";
import { enqueueFollowupRun, refreshQueuedFollowupSession } from "./queue.js";
@@ -310,10 +313,9 @@ export async function completeReplyAgentRun(input: {
runtimePolicySessionKey,
opts,
});
const finalDeliveryText = buildPendingFinalDeliveryText(finalPayloads);
// #85714: warn only for unusually substantive private final text. In
// message_tool_only, no tool call can be intentional silence, and
// finalDeliveryText also includes verbose/status/usage metadata.
// final payloads also include verbose/status/usage metadata.
const assistantFinalText = normalizeAssistantFinalDeliveryText(
typeof runResult.meta?.finalAssistantVisibleText === "string"
? runResult.meta.finalAssistantVisibleText
@@ -357,7 +359,12 @@ export async function completeReplyAgentRun(input: {
finalPayloads = [...finalPayloads, buildStrandedReplyDeliveryFailurePayload()];
}
}
const pendingText = sourceReplyPolicy.suppressDelivery ? "" : finalDeliveryText;
const recoverablePendingFinalText = buildRecoverablePendingFinalDeliveryText(
normalizePendingFinalRecoveryPayloads(finalPayloads),
);
const pendingText = sourceReplyPolicy.suppressDelivery
? ""
: (recoverablePendingFinalText ?? "");
const heartbeatAckMaxChars = DEFAULT_HEARTBEAT_ACK_MAX_CHARS;
const resolvedPendingText = isHeartbeat
? (() => {
@@ -368,17 +375,30 @@ export async function completeReplyAgentRun(input: {
return stripped.shouldSkip ? "" : stripped.text || pendingText;
})()
: pendingText;
if (resolvedPendingText) {
const sendableFinalPayloads = sourceReplyPolicy.suppressDelivery
? []
: finalPayloads.filter(
(payload) => normalizePendingFinalDeliveryPayloads([payload]).length > 0,
);
if (sendableFinalPayloads.length > 0) {
const pendingFinalDeliveryIntentId = crypto.randomUUID();
for (const payload of finalPayloads) {
const expectedSessionId = activeSessionEntry?.sessionId ?? followupRun.run.sessionId;
const pendingFinalDeliveries = sendableFinalPayloads.map((payload) => {
const deliveryId = crypto.randomUUID();
setReplyPayloadMetadata(payload, {
pendingFinalDeliveryIntentId,
pendingFinalDeliveryRetryText: resolvePendingFinalDeliveryRetryText({
isHeartbeat,
payload,
}),
pendingFinalDeliveryCompletion: {
deliveryId,
intentId: pendingFinalDeliveryIntentId,
...(activeSessionEntry?.restartRecoveryDeliveryRunId
? { recoveryRunId: activeSessionEntry.restartRecoveryDeliveryRunId }
: {}),
sessionId: expectedSessionId,
sessionKey,
storePath,
},
});
}
return { id: deliveryId, state: "prepared" as const };
});
const pendingFinalDeliveryContext = resolveReplyRunDeliveryContext({
cfg,
sessionCtx,
@@ -387,7 +407,6 @@ export async function completeReplyAgentRun(input: {
runtimePolicySessionKey,
opts,
});
const expectedSessionId = activeSessionEntry?.sessionId ?? followupRun.run.sessionId;
// A reset can rebind the key while the model runs; its replacement must
// never inherit the old run's final or advertise an uncommitted intent.
const persistedPendingFinalDelivery = await updateSessionEntry(
@@ -396,9 +415,11 @@ export async function completeReplyAgentRun(input: {
entry.sessionId === expectedSessionId
? {
pendingFinalDelivery: {
kind: "replayable" as const,
text: resolvedPendingText,
...(resolvedPendingText
? { kind: "replayable" as const, text: resolvedPendingText }
: { kind: "transport-only" as const }),
intentId: pendingFinalDeliveryIntentId,
deliveries: pendingFinalDeliveries,
context: pendingFinalDeliveryContext,
createdAt: Date.now(),
},
@@ -1588,16 +1588,84 @@ describe("runReplyAgent pending final delivery capture", () => {
kind: "replayable",
text: "visible final",
intentId: expect.any(String),
deliveries: [{ id: expect.any(String), state: "prepared" }],
});
const visiblePayload = (Array.isArray(result) ? result : [result]).find(
(payload) => payload?.text === "visible final",
);
expect(getReplyPayloadMetadata(visiblePayload ?? {})).toMatchObject({
pendingFinalDeliveryIntentId: stored.pendingFinalDelivery?.intentId,
pendingFinalDeliveryRetryText: "visible final",
pendingFinalDeliveryCompletion: {
deliveryId: stored.pendingFinalDelivery?.deliveries?.[0]?.id,
intentId: stored.pendingFinalDelivery?.intentId,
sessionId: "session",
sessionKey: "main",
storePath,
},
});
});
it("owns a media-only final with its complete replay directive", async () => {
const { sessionEntry, sessionStore, storePath } = await makeSessionFixture();
state.runEmbeddedAgentMock.mockResolvedValueOnce({
payloads: [{ mediaUrl: "https://example.test/final.png" }],
meta: {},
});
const { run } = createMinimalRun({
sessionEntry,
sessionStore,
sessionKey: "main",
storePath,
});
const result = await run();
const stored = await readStoredMainSession(storePath);
expect(stored.pendingFinalDelivery).toMatchObject({
kind: "replayable",
text: "MEDIA:https://example.test/final.png",
intentId: expect.any(String),
deliveries: [{ id: expect.any(String), state: "prepared" }],
});
const payload = Array.isArray(result) ? result[0] : result;
expect(getReplyPayloadMetadata(payload ?? {})).toMatchObject({
pendingFinalDeliveryCompletion: {
deliveryId: stored.pendingFinalDelivery?.deliveries?.[0]?.id,
intentId: stored.pendingFinalDelivery?.intentId,
},
});
});
it("owns mixed text and media finals without replaying a partial aggregate", async () => {
const { sessionEntry, sessionStore, storePath } = await makeSessionFixture();
state.runEmbeddedAgentMock.mockResolvedValueOnce({
payloads: [{ text: "visible text" }, { mediaUrl: "https://example.test/final.png" }],
meta: {},
});
const { run } = createMinimalRun({
sessionEntry,
sessionStore,
sessionKey: "main",
storePath,
});
const result = await run();
const payloads = Array.isArray(result) ? result : [result];
const stored = await readStoredMainSession(storePath);
expect(stored.pendingFinalDelivery).toMatchObject({
kind: "transport-only",
deliveries: [
{ id: expect.any(String), state: "prepared" },
{ id: expect.any(String), state: "prepared" },
],
});
expect(stored.pendingFinalDelivery).not.toHaveProperty("text");
expect(
payloads.map(
(payload) =>
getReplyPayloadMetadata(payload ?? {})?.pendingFinalDeliveryCompletion?.deliveryId,
),
).toEqual(stored.pendingFinalDelivery?.deliveries?.map(({ id }) => id));
});
it("persists canonical SQLite pending final delivery after its intent commits", async () => {
const sessionKey = "agent:main:main";
const { sessionEntry, sessionStore, storePath } = await makeSessionFixture({}, sessionKey);
@@ -1620,13 +1688,19 @@ describe("runReplyAgent pending final delivery capture", () => {
kind: "replayable",
intentId: expect.any(String),
text: "visible canonical final",
deliveries: [{ id: expect.any(String), state: "prepared" }],
},
sessionId: "session",
});
const visiblePayload = Array.isArray(result) ? result[0] : result;
expect(getReplyPayloadMetadata(visiblePayload ?? {})).toMatchObject({
pendingFinalDeliveryIntentId: stored?.pendingFinalDelivery?.intentId,
pendingFinalDeliveryRetryText: "visible canonical final",
pendingFinalDeliveryCompletion: {
deliveryId: stored?.pendingFinalDelivery?.deliveries?.[0]?.id,
intentId: stored?.pendingFinalDelivery?.intentId,
sessionId: "session",
sessionKey,
storePath,
},
});
expect(state.runEmbeddedAgentMock).toHaveBeenCalledOnce();
});
@@ -2857,11 +2931,17 @@ describe("runReplyAgent pending final delivery capture", () => {
expect(stored.pendingFinalDelivery).toMatchObject({
kind: "replayable",
text: longRemainder,
deliveries: [{ id: expect.any(String), state: "prepared" }],
});
const payload = Array.isArray(result) ? result[0] : result;
expect(getReplyPayloadMetadata(payload ?? {})).toMatchObject({
pendingFinalDeliveryIntentId: stored.pendingFinalDelivery?.intentId,
pendingFinalDeliveryRetryText: longRemainder,
pendingFinalDeliveryCompletion: {
deliveryId: stored.pendingFinalDelivery?.deliveries?.[0]?.id,
intentId: stored.pendingFinalDelivery?.intentId,
sessionId: "session",
sessionKey: "main",
storePath,
},
});
});
});
+239 -1
View File
@@ -1,5 +1,12 @@
// Tests before-deliver hook ordering and payload mutation behavior.
import { describe, expect, it } from "vitest";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import { createDeferred } from "../../../test/helpers/promise.js";
import { createDirectPendingFinalCustody } from "../../channels/turn/direct-delivery-custody.js";
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
import type { InternalSessionEntry } from "../../config/sessions/types.js";
import { getReplyPayloadMetadata, setReplyPayloadMetadata } from "../reply-payload.js";
import type { ReplyPayload } from "../types.js";
import {
@@ -9,6 +16,40 @@ import {
createReplyDispatcher,
} from "./reply-dispatcher.js";
async function makePendingFinalFixture() {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-dispatcher-pending-final-"));
const storePath = path.join(tmpDir, "sessions.json");
const sessionKey = "agent:main:telegram:direct:123";
await replaceSessionEntry(
{ sessionKey, storePath },
{
sessionId: "session-1",
status: "running",
updatedAt: Date.now(),
pendingFinalDelivery: {
kind: "replayable",
text: "final answer",
createdAt: Date.now(),
intentId: "intent-1",
deliveries: [{ id: "delivery-1", state: "prepared" }],
},
},
);
const payload = setReplyPayloadMetadata(
{ text: "final answer" },
{
pendingFinalDeliveryCompletion: {
deliveryId: "delivery-1",
intentId: "intent-1",
sessionId: "session-1",
sessionKey,
storePath,
},
},
);
return { payload, sessionKey, storePath, tmpDir };
}
describe("beforeDeliver in reply dispatcher", () => {
it("delivers the attached fallback when the primary payload is cancelled", async () => {
const delivered: string[] = [];
@@ -301,4 +342,201 @@ describe("beforeDeliver in reply dispatcher", () => {
expect(delivered).toEqual(["plain reply"]);
});
it("records direct-delivery custody before waiting for the channel provider", async () => {
const fixture = await makePendingFinalFixture();
const enteredProvider = createDeferred();
const releaseProvider = createDeferred();
try {
const dispatcher = createReplyDispatcher({
deliver: async () => {
enteredProvider.resolve();
await releaseProvider.promise;
},
});
dispatcher.sendFinalReply(fixture.payload);
dispatcher.markComplete();
await enteredProvider.promise;
expect(
(
loadSessionEntry({
sessionKey: fixture.sessionKey,
storePath: fixture.storePath,
}) as InternalSessionEntry
)?.pendingFinalDelivery?.deliveries,
).toEqual([{ id: "delivery-1", state: "queued" }]);
releaseProvider.resolve();
await dispatcher.waitForIdle();
expect(
(
loadSessionEntry({
sessionKey: fixture.sessionKey,
storePath: fixture.storePath,
}) as InternalSessionEntry
)?.pendingFinalDelivery?.deliveries,
).toEqual([{ id: "delivery-1", state: "delivered" }]);
} finally {
releaseProvider.resolve();
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
}
});
it.each([
{
label: "proven pre-send failure",
error: () =>
Object.assign(new Error("connect failed"), { code: "ECONNREFUSED", syscall: "connect" }),
expected: "prepared",
},
{
label: "ambiguous provider failure",
error: () => new Error("send outcome unknown"),
expected: "unknown",
},
] as const)("records $label before reporting the error", async ({ error, expected }) => {
const fixture = await makePendingFinalFixture();
try {
const dispatcher = createReplyDispatcher({
deliver: async () => {
throw error();
},
});
dispatcher.sendFinalReply(fixture.payload);
dispatcher.markComplete();
await dispatcher.waitForIdle();
expect(
(
loadSessionEntry({
sessionKey: fixture.sessionKey,
storePath: fixture.storePath,
}) as InternalSessionEntry
)?.pendingFinalDelivery?.deliveries,
).toEqual([{ id: "delivery-1", state: expected }]);
} finally {
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
}
});
it("restores prepared custody when a pre-I/O admitted send proves no-send", async () => {
const fixture = await makePendingFinalFixture();
try {
const dispatcher = createReplyDispatcher({
deliver: async (payload) => {
// Mirror the channel-turn direct path: custody escalates queued→unknown
// immediately before wire I/O, then the provider proves no send happened.
const custody = createDirectPendingFinalCustody(payload);
await custody?.onPlatformSendDispatch();
throw Object.assign(new Error("connect failed"), {
code: "ECONNREFUSED",
syscall: "connect",
});
},
});
dispatcher.sendFinalReply(fixture.payload);
dispatcher.markComplete();
await dispatcher.waitForIdle();
expect(
(
loadSessionEntry({
sessionKey: fixture.sessionKey,
storePath: fixture.storePath,
}) as InternalSessionEntry
)?.pendingFinalDelivery?.deliveries,
).toEqual([{ id: "delivery-1", state: "prepared" }]);
} finally {
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
}
});
it("suppresses a second direct call after the exact delivery is terminal", async () => {
const fixture = await makePendingFinalFixture();
const deliver = vi.fn(async () => {});
try {
const first = createReplyDispatcher({ deliver });
first.sendFinalReply(fixture.payload);
first.markComplete();
await first.waitForIdle();
const second = createReplyDispatcher({ deliver });
second.sendFinalReply(fixture.payload);
second.markComplete();
await second.waitForIdle();
expect(deliver).toHaveBeenCalledOnce();
expect(second.getCancelledCounts?.().final).toBe(1);
} finally {
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
}
});
it("suppresses a direct call whose persisted owner was replaced", async () => {
const fixture = await makePendingFinalFixture();
const current = loadSessionEntry({
sessionKey: fixture.sessionKey,
storePath: fixture.storePath,
}) as InternalSessionEntry;
await replaceSessionEntry(
{ sessionKey: fixture.sessionKey, storePath: fixture.storePath },
{
...current,
pendingFinalDelivery: {
...current.pendingFinalDelivery!,
intentId: "replacement-intent",
},
},
);
const deliver = vi.fn(async () => {});
try {
const dispatcher = createReplyDispatcher({ deliver });
dispatcher.sendFinalReply(fixture.payload);
dispatcher.markComplete();
await dispatcher.waitForIdle();
expect(deliver).not.toHaveBeenCalled();
expect(dispatcher.getCancelledCounts?.().final).toBe(1);
} finally {
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
}
});
it("records policy suppression before awaiting cancellation observers", async () => {
const fixture = await makePendingFinalFixture();
const observerStarted = createDeferred();
const releaseObserver = createDeferred();
try {
const dispatcher = createReplyDispatcher({
beforeDeliver: () => null,
deliver: async () => {},
onBeforeDeliverCancelled: async () => {
observerStarted.resolve();
await releaseObserver.promise;
},
});
dispatcher.sendFinalReply(fixture.payload);
dispatcher.markComplete();
await observerStarted.promise;
expect(
(
loadSessionEntry({
sessionKey: fixture.sessionKey,
storePath: fixture.storePath,
}) as InternalSessionEntry
)?.pendingFinalDelivery?.deliveries,
).toEqual([{ id: "delivery-1", state: "suppressed" }]);
releaseObserver.resolve();
await dispatcher.waitForIdle();
} finally {
releaseObserver.resolve();
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
}
});
});
@@ -21,8 +21,7 @@ import {
} from "./dispatch-from-config.payloads.js";
import {
clearPendingFinalDeliveryAfterSuccess,
capturePendingFinalDeliveryIdentity,
reconcilePendingFinalDeliveryAfterSettlement,
suppressPendingFinalDelivery,
} from "./dispatch-from-config.pending-final.js";
import type { ReplyDispatchDeliveryOutcome } from "./reply-dispatcher.js";
@@ -48,26 +47,15 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
sendPolicyDenied,
sessionAgentId,
sessionKey,
sessionStoreEntry,
suppressDelivery,
throwIfDispatchOperationAborted,
turnLedger,
waitForPendingDirectBlockReplyDelivery,
} = state;
const replies = replyResult ? (Array.isArray(replyResult) ? replyResult : [replyResult]) : [];
const pendingFinalDelivery = {
storePath: sessionStoreEntry.storePath,
sessionKey: sessionStoreEntry.sessionKey ?? sessionKey,
};
const replyPendingIntentIds = new Set(
replies
.map((reply) => getReplyPayloadMetadata(reply)?.pendingFinalDeliveryIntentId)
.filter((intentId): intentId is string => Boolean(intentId)),
);
const pendingFinalDeliveryIdentity = capturePendingFinalDeliveryIdentity({
...pendingFinalDelivery,
intentId: replyPendingIntentIds.size === 1 ? [...replyPendingIntentIds][0] : undefined,
});
const pendingFinalDeliveryIdentity = replies
.map((reply) => getReplyPayloadMetadata(reply)?.pendingFinalDeliveryCompletion)
.find((completion) => completion !== undefined);
// Final delivery is outside the progress wrappers. Wait until every source-ordered callback
// has at least started so a delayed tool/reasoning transition cannot appear after the final.
if (state.preserveProgressCallbackStartOrder) {
@@ -84,10 +72,7 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
let routedFinalCount = 0;
let attemptedFinalDelivery = false;
let finalDeliveryFailed = false;
const finalDeliveries: Array<{
outcome: Promise<ReplyDispatchDeliveryOutcome>;
payload: ReplyPayload;
}> = [];
const finalDeliveries: Promise<ReplyDispatchDeliveryOutcome>[] = [];
let allQueuedFinalsObserved = true;
const sentFinalPayloadDedupeKeys = new Set<string>();
let deferredTtsTextPending = state.progressState.accumulatedBlockTtsText;
@@ -96,9 +81,11 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
// Durable reasoning is a channel-owned lane; generic channels keep the
// historical suppression unless they explicitly opt in.
if (reply.isReasoning === true && !state.reasoningPayloadsEnabled) {
await suppressPendingFinalDelivery(reply);
continue;
}
if (reply.isCommentary === true && !state.commentaryPayloadsEnabled) {
await suppressPendingFinalDelivery(reply);
continue;
}
if (suppressDelivery && !shouldDeliverDespiteSourceReplySuppression(reply, state)) {
@@ -116,10 +103,12 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
].join(" "),
);
}
await suppressPendingFinalDelivery(reply);
continue;
}
const finalPayloadDedupeKey = createFinalDispatchPayloadDedupeKey(reply);
if (sentFinalPayloadDedupeKeys.has(finalPayloadDedupeKey)) {
await suppressPendingFinalDelivery(reply);
continue;
}
sentFinalPayloadDedupeKeys.add(finalPayloadDedupeKey);
@@ -141,6 +130,7 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
}
if (finalReply.dedupedAgainstBlock) {
// The delivering block already settled into the turn ledger.
await suppressPendingFinalDelivery(reply);
continue;
}
attemptedFinalDelivery = true;
@@ -148,7 +138,7 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
routedFinalCount += finalReply.routedFinalCount;
if (finalReply.queuedFinal) {
if (finalReply.dispatcherOutcome) {
finalDeliveries.push({ outcome: finalReply.dispatcherOutcome, payload: reply });
finalDeliveries.push(finalReply.dispatcherOutcome);
} else {
allQueuedFinalsObserved = false;
}
@@ -162,19 +152,9 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
if (queuedFinal && allQueuedFinalsObserved) {
// Delivery observers run from the queue itself, so direct low-level callers
// reconcile too; the settle task only makes lifecycle owners await it.
const reconcilePendingFinal = Promise.all(
finalDeliveries.map(async (delivery) => ({
outcome: await delivery.outcome,
payload: delivery.payload,
})),
)
.then(async (deliveries) => {
await reconcilePendingFinalDeliveryAfterSettlement({
...pendingFinalDelivery,
deliveries,
identity: pendingFinalDeliveryIdentity,
replies,
});
const reconcilePendingFinal = Promise.all(finalDeliveries)
.then(async () => {
await clearPendingFinalDeliveryAfterSuccess(pendingFinalDeliveryIdentity);
})
.catch((error: unknown) => {
logVerbose(
@@ -185,10 +165,7 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
} else {
// Routed delivery has a transport result already. Custom dispatchers that
// do not expose the core observer retain the legacy queue-admission behavior.
await clearPendingFinalDeliveryAfterSuccess({
...pendingFinalDelivery,
identity: pendingFinalDeliveryIdentity,
});
await clearPendingFinalDeliveryAfterSuccess(pendingFinalDeliveryIdentity);
}
// Register successful queued cleanup before honoring a late abort. The
// outer settle owner still runs it from finally (#89115).
@@ -4,11 +4,14 @@ import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
import type { ReplyPayload } from "../reply-payload.js";
import {
capturePendingFinalDeliveryIdentity,
getReplyPayloadMetadata,
setReplyPayloadMetadata,
type ReplyPayload,
} from "../reply-payload.js";
import {
clearPendingFinalDeliveryAfterSuccess,
reconcilePendingFinalDeliveryAfterSettlement,
suppressPendingFinalDelivery,
} from "./dispatch-from-config.pending-final.js";
import { retireTerminalRestartRecoverySourceClaim } from "./restart-recovery-claim.js";
@@ -28,6 +31,7 @@ describe("pending final delivery restart proof", () => {
async function writePendingFinal(
beforeAgentReplyState: "continue" | "handled-reply",
state: "prepared" | "delivered" = "delivered",
): Promise<void> {
const entry: SessionEntry = {
sessionId: "session",
@@ -40,6 +44,7 @@ describe("pending final delivery restart proof", () => {
text: "hook reply",
createdAt: 1,
intentId: "intent-1",
deliveries: [{ id: "delivery-1", state }],
},
restartRecoveryBeforeAgentReplyState: beforeAgentReplyState,
restartRecoveryForceSafeTools: beforeAgentReplyState === "handled-reply" ? true : undefined,
@@ -48,17 +53,28 @@ describe("pending final delivery restart proof", () => {
await replaceSessionEntry({ storePath, sessionKey }, entry);
}
function pendingFinalPayload(deliveryId = "delivery-1"): ReplyPayload {
const payload: ReplyPayload = { text: "hook reply" };
setReplyPayloadMetadata(payload, {
pendingFinalDeliveryCompletion: {
deliveryId,
intentId: "intent-1",
sessionId: "session",
sessionKey,
storePath,
},
});
return payload;
}
it.each(["continue", "handled-reply"] as const)(
"clears %s provenance only after the exact pending intent succeeds",
async (beforeAgentReplyState) => {
await writePendingFinal(beforeAgentReplyState);
const identity = capturePendingFinalDeliveryIdentity({
intentId: "intent-1",
sessionKey,
storePath,
});
const identity =
getReplyPayloadMetadata(pendingFinalPayload())?.pendingFinalDeliveryCompletion;
await clearPendingFinalDeliveryAfterSuccess({ identity, sessionKey, storePath });
await clearPendingFinalDeliveryAfterSuccess(identity);
const entry = loadSessionEntry({ sessionKey, storePath }) as SessionEntry | undefined;
expect(entry?.pendingFinalDelivery).toBeUndefined();
@@ -87,18 +103,25 @@ describe("pending final delivery restart proof", () => {
kind: "transport-only",
createdAt: Date.now(),
intentId: "intent-media",
deliveries: [{ id: "delivery-media", state: "delivered" }],
},
restartRecoveryBeforeAgentReplyState: "handled-unrecoverable",
restartRecoverySourceIngress: "channel",
};
await replaceSessionEntry({ storePath, sessionKey }, entry);
const identity = capturePendingFinalDeliveryIdentity({
intentId: "intent-media",
sessionKey,
storePath,
const payload: ReplyPayload = { mediaUrl: "https://example.test/image.png" };
setReplyPayloadMetadata(payload, {
pendingFinalDeliveryCompletion: {
deliveryId: "delivery-media",
intentId: "intent-media",
sessionId: "session",
sessionKey,
storePath,
},
});
const identity = getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion;
await clearPendingFinalDeliveryAfterSuccess({ identity, sessionKey, storePath });
await clearPendingFinalDeliveryAfterSuccess(identity);
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
status: "done",
@@ -109,32 +132,40 @@ describe("pending final delivery restart proof", () => {
).toBeUndefined();
});
it("keeps normal-turn provenance when transport fails before delivery", async () => {
await writePendingFinal("continue");
const identity = capturePendingFinalDeliveryIdentity({
intentId: "intent-1",
sessionKey,
storePath,
});
const payload: ReplyPayload = { text: "hook reply" };
await reconcilePendingFinalDeliveryAfterSettlement({
deliveries: [{ outcome: "failed-before-deliver", payload }],
identity,
replies: [payload],
sessionKey,
storePath,
});
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
pendingFinalDelivery: {
kind: "replayable",
text: "hook reply",
intentId: "intent-1",
it("clears a skipped turn only after every sendable final is suppressed", async () => {
await writePendingFinal("continue", "prepared");
await replaceSessionEntry(
{ storePath, sessionKey },
{
...(loadSessionEntry({ sessionKey, storePath }) as SessionEntry),
pendingFinalDelivery: {
kind: "replayable",
text: "hook reply",
createdAt: 1,
intentId: "intent-1",
deliveries: [
{ id: "delivery-1", state: "prepared" },
{ id: "delivery-2", state: "prepared" },
],
},
},
restartRecoveryBeforeAgentReplyState: "continue",
restartRecoverySourceIngress: "channel",
});
);
await suppressPendingFinalDelivery(pendingFinalPayload("delivery-1"));
expect(
(loadSessionEntry({ sessionKey, storePath }) as SessionEntry).pendingFinalDelivery
?.deliveries,
).toEqual([
{ id: "delivery-1", state: "suppressed" },
{ id: "delivery-2", state: "prepared" },
]);
await suppressPendingFinalDelivery(pendingFinalPayload("delivery-2"));
expect(
(loadSessionEntry({ sessionKey, storePath }) as SessionEntry).pendingFinalDelivery,
).toBeUndefined();
});
it("does not retire a source while its terminal provider outcome is unknown", async () => {
@@ -1,245 +1,73 @@
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { buildRestartRecoveryClaimCleanupPatch } from "../../config/sessions/restart-recovery-state.js";
import { updateSessionEntry } from "../../config/sessions/session-accessor.js";
import { settlePendingFinalDelivery } from "../../infra/outbound/delivery-completion.js";
import {
loadSessionEntryReadOnly,
updateSessionEntry,
} from "../../config/sessions/session-accessor.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
import type { ReplyPayload } from "../reply-payload.js";
import { getReplyPayloadMetadata } from "../reply-payload.js";
import {
buildPendingFinalDeliveryText,
sanitizePendingFinalDeliveryText,
} from "./pending-final-delivery.js";
import type { ReplyDispatchDeliveryOutcome } from "./reply-dispatcher.js";
getReplyPayloadMetadata,
type ReplyPayload,
type ReplyPayloadMetadata,
} from "../reply-payload.js";
type SettledFinalDelivery = {
outcome: ReplyDispatchDeliveryOutcome;
payload: ReplyPayload;
};
type PendingFinalDeliveryIdentity = NonNullable<
ReplyPayloadMetadata["pendingFinalDeliveryCompletion"]
>;
type PendingFinalDeliveryIdentity = {
createdAt?: number;
intentId?: string;
present: boolean;
text?: string;
};
function buildPendingFinalDeliveryCleanupPatch(entry: SessionEntry): Partial<SessionEntry> {
// An active receipt/claim may outlive outer reply settlement. Only claimless pending finals
// borrow hook provenance until their exact transport intent settles.
const clearsRestartRecoveryProof =
normalizeOptionalString(entry.restartRecoveryDeliveryRunId) === undefined;
const completesHookHandledTurn =
clearsRestartRecoveryProof &&
(entry.restartRecoveryBeforeAgentReplyState === "handled-reply" ||
entry.restartRecoveryBeforeAgentReplyState === "handled-unrecoverable");
const endedAt = completesHookHandledTurn ? Date.now() : undefined;
return {
pendingFinalDelivery: undefined,
...(clearsRestartRecoveryProof
? {
restartRecoveryBeforeAgentReplyState: undefined,
restartRecoverySourceIngress: undefined,
restartRecoveryForceSafeTools: undefined,
}
: {}),
...(endedAt !== undefined
? {
abortedLastRun: false,
endedAt,
lifecycleRunId: undefined,
runtimeMs:
typeof entry.startedAt === "number"
? Math.max(0, endedAt - entry.startedAt)
: undefined,
status: "done" as const,
}
: {}),
};
export async function suppressPendingFinalDelivery(payload: ReplyPayload): Promise<void> {
const completion = getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion;
if (completion) {
await settlePendingFinalDelivery({ kind: "pending-final", ...completion }, "suppressed", [
"prepared",
]);
await clearPendingFinalDeliveryAfterSuccess(completion);
}
}
function matchesPendingFinalDeliveryIdentity(
entry: SessionEntry,
expected: PendingFinalDeliveryIdentity,
): boolean {
const pending = entry.pendingFinalDelivery;
const currentPresent = pending !== undefined;
if (currentPresent !== expected.present) {
return false;
}
if (expected.intentId) {
return pending?.intentId === expected.intentId;
}
return (
pending?.createdAt === expected.createdAt &&
(pending?.kind === "replayable" ? pending.text : undefined) === expected.text
);
}
export async function clearPendingFinalDeliveryAfterSuccess(params: {
identity?: PendingFinalDeliveryIdentity;
storePath?: string;
sessionKey?: string;
}): Promise<void> {
const identity = params.identity;
if (!params.storePath || !params.sessionKey || !identity?.present) {
export async function clearPendingFinalDeliveryAfterSuccess(
identity?: PendingFinalDeliveryIdentity,
): Promise<void> {
if (!identity) {
return;
}
await updateSessionEntry(
{ storePath: params.storePath, sessionKey: params.sessionKey },
async (entry) => {
if (!matchesPendingFinalDeliveryIdentity(entry, identity)) {
return null;
}
if (!entry.pendingFinalDelivery) {
return null;
}
return {
...buildPendingFinalDeliveryCleanupPatch(entry),
updatedAt: Date.now(),
};
},
{ skipMaintenance: true, takeCacheOwnership: true },
);
}
export function capturePendingFinalDeliveryIdentity(params: {
intentId?: string;
storePath?: string;
sessionKey?: string;
}): PendingFinalDeliveryIdentity | undefined {
if (!params.storePath || !params.sessionKey) {
return undefined;
}
try {
const entry = loadSessionEntryReadOnly({
storePath: params.storePath,
sessionKey: params.sessionKey,
hydrateSkillPromptRefs: false,
readConsistency: "latest",
});
const pending = entry?.pendingFinalDelivery;
if (params.intentId && pending?.intentId !== params.intentId) {
return { present: false };
}
return {
present: pending !== undefined,
intentId: params.intentId ?? pending?.intentId,
createdAt: pending?.createdAt,
text: pending?.kind === "replayable" ? pending.text : undefined,
};
} catch {
return params.intentId ? { present: true, intentId: params.intentId } : undefined;
}
}
function buildPendingFinalDeliveryRetryText(payloads: ReplyPayload[]): string {
return sanitizePendingFinalDeliveryText(
payloads
.map(
(payload) =>
getReplyPayloadMetadata(payload)?.pendingFinalDeliveryRetryText ??
buildPendingFinalDeliveryText([payload]),
)
.filter(Boolean)
.join("\n\n"),
);
}
function resolvePendingFinalDeliveryPayloads(params: {
intentId?: string;
pendingText: string;
replies: ReplyPayload[];
}): ReplyPayload[] | undefined {
const intentReplies = params.intentId
? params.replies.filter((reply) => {
const metadata = getReplyPayloadMetadata(reply);
return (
metadata?.pendingFinalDeliveryIntentId === params.intentId &&
metadata?.pendingFinalDeliveryRetryText !== undefined
);
})
: [];
const intentContributors = intentReplies.filter(
(reply) => getReplyPayloadMetadata(reply)?.pendingFinalDeliveryRetryText,
);
const intentText = buildPendingFinalDeliveryRetryText(intentContributors);
if (
intentReplies.length > 0 &&
intentText.replace(/\s+/g, " ").trim() === params.pendingText.replace(/\s+/g, " ").trim()
) {
return intentContributors;
}
const contributingReplies = params.replies.filter(
(reply) => buildPendingFinalDeliveryText([reply]) !== "",
);
if (buildPendingFinalDeliveryText(contributingReplies) === params.pendingText) {
return contributingReplies;
}
const exactMatches = contributingReplies.filter(
(reply) => buildPendingFinalDeliveryText([reply]) === params.pendingText,
);
return exactMatches.length === 1 ? exactMatches : undefined;
}
export async function reconcilePendingFinalDeliveryAfterSettlement(params: {
deliveries: SettledFinalDelivery[];
identity?: PendingFinalDeliveryIdentity;
replies: ReplyPayload[];
storePath?: string;
sessionKey?: string;
}): Promise<void> {
const identity = params.identity;
if (!params.storePath || !params.sessionKey || !identity?.present) {
return;
}
await updateSessionEntry(
{ storePath: params.storePath, sessionKey: params.sessionKey },
async (entry) => {
if (!matchesPendingFinalDeliveryIdentity(entry, identity)) {
return null;
}
const pending = entry.pendingFinalDelivery;
if (!pending) {
return null;
}
const pendingPayloads =
pending.kind === "replayable"
? resolvePendingFinalDeliveryPayloads({
intentId: identity.intentId,
pendingText: pending.text,
replies: params.replies,
})
: undefined;
const pendingPayloadSet = pendingPayloads ? new Set(pendingPayloads) : undefined;
const relevantDeliveries = pendingPayloadSet
? params.deliveries.filter((delivery) => pendingPayloadSet.has(delivery.payload))
: params.deliveries;
const ownsEveryPendingPayload =
!pendingPayloadSet || relevantDeliveries.length === pendingPayloadSet.size;
const failedBeforeDeliver = relevantDeliveries.filter(
(delivery) => delivery.outcome === "failed-before-deliver",
);
{ storePath: identity.storePath, sessionKey: identity.sessionKey },
(entry) => {
const recoveryRunId = normalizeOptionalString(entry.restartRecoveryDeliveryRunId);
const deliveries = entry.pendingFinalDelivery?.deliveries;
if (
relevantDeliveries.length > 0 &&
failedBeforeDeliver.length === relevantDeliveries.length
entry.sessionId !== identity.sessionId ||
entry.pendingFinalDelivery?.intentId !== identity.intentId ||
!deliveries?.length ||
!deliveries.every(({ state }) => state === "delivered" || state === "suppressed") ||
(recoveryRunId !== undefined && recoveryRunId !== identity.recoveryRunId)
) {
return null;
}
if (pendingPayloadSet && ownsEveryPendingPayload && failedBeforeDeliver.length > 0) {
const retryText = buildPendingFinalDeliveryRetryText(
failedBeforeDeliver.map((delivery) => delivery.payload),
);
if (retryText && pending.kind === "replayable") {
return {
pendingFinalDelivery: { ...pending, text: retryText },
updatedAt: Date.now(),
};
}
}
const completesHookTurn =
recoveryRunId === undefined &&
(entry.restartRecoveryBeforeAgentReplyState === "handled-reply" ||
entry.restartRecoveryBeforeAgentReplyState === "handled-unrecoverable");
const endedAt = completesHookTurn ? Date.now() : undefined;
return {
...buildPendingFinalDeliveryCleanupPatch(entry),
...(recoveryRunId
? buildRestartRecoveryClaimCleanupPatch({ entry, recordTerminalSource: true })
: {
restartRecoveryBeforeAgentReplyState: undefined,
restartRecoverySourceIngress: undefined,
restartRecoveryForceSafeTools: undefined,
}),
pendingFinalDelivery: undefined,
...(endedAt === undefined
? {}
: {
abortedLastRun: false,
endedAt,
lifecycleRunId: undefined,
runtimeMs:
typeof entry.startedAt === "number"
? Math.max(0, endedAt - entry.startedAt)
: undefined,
status: "done" as const,
}),
updatedAt: Date.now(),
};
},
@@ -2,6 +2,10 @@
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { createDeferred } from "../../../test/helpers/promise.js";
import { clearAgentHarnesses } from "../../agents/harness/registry.js";
import {
OutboundDeliveryError,
PlatformMessageNotDispatchedError,
} from "../../infra/outbound/deliver-types.js";
import type { PluginHookReplyDispatchResult } from "../../plugins/hooks.test-fixtures.js";
import { getPluginRuntimeGatewayRequestScope } from "../../plugins/runtime/gateway-request-scope.js";
import { createInternalHookEventPayload } from "../../test-utils/internal-hook-event-payload.js";
@@ -55,9 +59,42 @@ function firstReplyDispatchCall() {
function pendingFinalDelivery(
text: string,
overrides: { createdAt?: number; context?: Record<string, unknown>; intentId?: string } = {},
overrides: {
createdAt?: number;
context?: Record<string, unknown>;
deliveries?: Array<{
id: string;
state: "prepared" | "queued" | "delivered" | "suppressed" | "unknown";
}>;
intentId?: string;
} = {},
) {
return { kind: "replayable" as const, text, createdAt: 1, ...overrides };
return {
kind: "replayable" as const,
text,
createdAt: 1,
intentId: "intent-1",
deliveries: [{ id: "delivery-1", state: "prepared" as const }],
...overrides,
};
}
function pendingFinalReply(
text: string,
overrides: { deliveryId?: string; intentId?: string } = {},
): ReplyPayload {
return setReplyPayloadMetadata(
{ text },
{
pendingFinalDeliveryCompletion: {
deliveryId: overrides.deliveryId ?? "delivery-1",
intentId: overrides.intentId ?? "intent-1",
sessionId: "session-1",
sessionKey: "agent:test:session",
storePath: "/tmp/mock-sessions.json",
},
},
);
}
describe("dispatchReplyFromConfig reply_dispatch hook", () => {
@@ -217,6 +254,93 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
});
});
it("clears pending final delivery after final dispatch succeeds", async () => {
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pendingFinalDelivery("durable reply", {
context: { source: "heartbeat" },
}),
};
sessionStoreMocks.loadSessionStore.mockClear();
mocks.routeReply.mockResolvedValue({ ok: true, delivered: true, messageId: "mock" });
const deliver = vi.fn().mockResolvedValue(undefined);
const dispatcher = createReplyDispatcher({ deliver });
const result = await dispatchReplyFromConfig({
ctx: createHookCtx(),
cfg: emptyConfig,
dispatcher,
replyResolver: async () => pendingFinalReply("durable reply"),
});
await dispatcher.waitForIdle();
await vi.waitFor(() => {
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined();
});
expect(result.queuedFinal).toBe(true);
expect(sessionStoreMocks.loadSessionStoreEntry).toHaveBeenCalledWith({
agentId: "test",
storePath: "/tmp/mock-sessions.json",
sessionKey: "agent:test:session",
readConsistency: "latest",
clone: false,
});
expect(sessionStoreMocks.loadSessionStore).not.toHaveBeenCalled();
expect(deliver).toHaveBeenCalledOnce();
expect(sessionStoreMocks.updateSessionEntry).toHaveBeenCalledTimes(3);
});
it("clears pending final delivery when abort fires after a successful final send (#89115)", async () => {
// Regression for #89115: an abort that lands after the final reply has
// shipped (here, during sendFinalReply) must still clear the pending-final
// bookkeeping — otherwise pendingFinalDelivery stays true and the get-reply
// redelivery short-circuit silently blocks every later inbound.
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pendingFinalDelivery("durable reply", {
context: { source: "heartbeat" },
intentId: "intent-89115",
}),
};
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
existing: sessionStoreMocks.currentEntry,
});
const abortController = new AbortController();
const deliver = vi.fn().mockResolvedValue(undefined);
const dispatcher = createReplyDispatcher({ deliver });
const sendFinalReply = dispatcher.sendFinalReply.bind(dispatcher);
vi.spyOn(dispatcher, "sendFinalReply").mockImplementation((payload) => {
const queued = sendFinalReply(payload);
abortController.abort();
return queued;
});
const result = await withReplyDispatcher({
dispatcher,
run: () =>
dispatchReplyFromConfig({
ctx: createHookCtx(),
cfg: emptyConfig,
dispatcher,
replyOptions: { abortSignal: abortController.signal },
replyResolver: async () =>
pendingFinalReply("durable reply", { intentId: "intent-89115" }),
}),
});
// Abort landed after delivery: the run is still surfaced as aborted
// (queuedFinal:false), but the pending-final state is fully cleared.
expect(dispatcher.sendFinalReply).toHaveBeenCalledOnce();
expect(deliver).toHaveBeenCalledOnce();
expect(result.queuedFinal).toBe(false);
expect(sessionStoreMocks.updateSessionEntry).toHaveBeenCalledTimes(3);
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined();
});
it("preserves pending final delivery when final dispatch fails", async () => {
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
@@ -248,6 +372,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
try {
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pendingFinalDelivery("durable reply", {
context: { channel: "whatsapp", to: "+1000" },
@@ -273,7 +398,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
ctx: createHookCtx(),
cfg: emptyConfig,
dispatcher,
replyResolver: async () => ({ text: "durable reply" }),
replyResolver: async () => pendingFinalReply("durable reply"),
}),
});
await hookStarted.promise;
@@ -299,11 +424,66 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
}
});
it("clears pending final delivery when a later queued final succeeds", async () => {
vi.useFakeTimers();
try {
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pendingFinalDelivery("durable reply"),
};
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
existing: sessionStoreMocks.currentEntry,
});
const hookStarted = createDeferred();
const deliver = vi.fn().mockResolvedValue(undefined);
let hookCalls = 0;
const dispatcher = createReplyDispatcher({
deliver,
beforeDeliver: (payload) => {
hookCalls += 1;
if (hookCalls === 1) {
hookStarted.resolve();
return new Promise<never>(() => {});
}
return payload;
},
});
const resultPromise = withReplyDispatcher({
dispatcher,
run: () =>
dispatchReplyFromConfig({
ctx: createHookCtx(),
cfg: emptyConfig,
dispatcher,
replyResolver: async () => [{ text: "first" }, pendingFinalReply("durable reply")],
}),
});
await hookStarted.promise;
await vi.advanceTimersByTimeAsync(15_000);
await resultPromise;
expect(deliver).toHaveBeenCalledOnce();
expect(deliver).toHaveBeenCalledWith(
expect.objectContaining({ text: "durable reply" }),
expect.objectContaining({ kind: "final" }),
);
expect(dispatcher.getFailedCounts?.()).toEqual({ tool: 0, block: 0, final: 1 });
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined();
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.useRealTimers();
}
});
it("preserves the durable final when an earlier auxiliary final succeeds", async () => {
vi.useFakeTimers();
try {
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pendingFinalDelivery("durable reply"),
};
@@ -332,7 +512,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
ctx: createHookCtx(),
cfg: emptyConfig,
dispatcher,
replyResolver: async () => [{ text: "auxiliary" }, { text: "durable reply" }],
replyResolver: async () => [{ text: "auxiliary" }, pendingFinalReply("durable reply")],
}),
});
await hookStarted.promise;
@@ -353,11 +533,74 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
}
});
it("records each pending-final delivery without rewriting aggregate text", async () => {
vi.useFakeTimers();
try {
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pendingFinalDelivery("auxiliary\n\ndurable reply", {
deliveries: [
{ id: "delivery-auxiliary", state: "prepared" },
{ id: "delivery-durable", state: "prepared" },
],
}),
};
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
existing: sessionStoreMocks.currentEntry,
});
const hookStarted = createDeferred();
let hookCalls = 0;
const dispatcher = createReplyDispatcher({
deliver: vi.fn().mockResolvedValue(undefined),
beforeDeliver: (payload) => {
hookCalls += 1;
if (hookCalls === 2) {
hookStarted.resolve();
return new Promise<never>(() => {});
}
return payload;
},
});
const resultPromise = withReplyDispatcher({
dispatcher,
run: () =>
dispatchReplyFromConfig({
ctx: createHookCtx(),
cfg: emptyConfig,
dispatcher,
replyResolver: async () => [
pendingFinalReply("auxiliary", { deliveryId: "delivery-auxiliary" }),
pendingFinalReply("durable reply", { deliveryId: "delivery-durable" }),
],
}),
});
await hookStarted.promise;
await vi.advanceTimersByTimeAsync(15_000);
await resultPromise;
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toEqual(
pendingFinalDelivery("auxiliary\n\ndurable reply", {
deliveries: [
{ id: "delivery-auxiliary", state: "delivered" },
{ id: "delivery-durable", state: "prepared" },
],
}),
);
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.useRealTimers();
}
});
it("does not let an older settlement rewrite a newer pending-final intent", async () => {
vi.useFakeTimers();
try {
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pendingFinalDelivery("older reply", { intentId: "older-intent" }),
};
@@ -381,10 +624,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
cfg: emptyConfig,
dispatcher,
replyResolver: async () =>
setReplyPayloadMetadata(
{ text: "older reply" },
{ pendingFinalDeliveryIntentId: "older-intent" },
),
pendingFinalReply("older reply", { intentId: "older-intent" }),
}),
});
await hookStarted.promise;
@@ -407,6 +647,107 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
}
});
const createNoSendFailure = (retryable = true) =>
new PlatformMessageNotDispatchedError("offline", { cause: new Error("offline"), retryable });
const wrapDeliveryFailure = (cause: unknown) =>
new OutboundDeliveryError("delivery failed", { cause });
const refused = Object.assign(new Error(), {
code: "ECONNREFUSED",
syscall: "connect",
});
const createPartialDelivery = () =>
Object.assign(new Error("partial delivery", { cause: createNoSendFailure() }), {
code: "CHANNEL_PARTIAL_DELIVERY",
deliveryResult: { visibleReplySent: true },
});
it.each([
["direct retryable provider proof", createNoSendFailure(), true],
["wrapped retryable provider proof", wrapDeliveryFailure(createNoSendFailure()), true],
["wrapped pre-connect ECONNREFUSED proof", wrapDeliveryFailure(refused), true],
["permanent provider rejection", createNoSendFailure(false), false],
[
"partial outbound delivery",
Object.assign(wrapDeliveryFailure(createNoSendFailure()), { sentBeforeError: true }),
false,
],
["nested partial envelope", new Error("partial", { cause: createPartialDelivery() }), false],
["aggregate partial envelope", new AggregateError([createPartialDelivery()]), false],
["observer-attached delivery evidence", createNoSendFailure(), true],
["ambiguous transport failure", new Error("transport failed"), false],
] as const)("reconciles pending final delivery after %s", async (name, error, preserve) => {
hookMocks.runner.hasHooks.mockReturnValue(false);
const pending = pendingFinalDelivery("recoverable final reply");
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pending,
};
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
existing: sessionStoreMocks.currentEntry,
});
const dispatcher = createReplyDispatcher({
deliver: async () => {
throw error;
},
onError: () => {
if (name.startsWith("observer")) {
Object.assign(error, { visibleReplySent: true });
}
},
});
await withReplyDispatcher({
dispatcher,
run: () =>
dispatchReplyFromConfig({
ctx: createHookCtx(),
cfg: emptyConfig,
dispatcher,
replyResolver: async () => pendingFinalReply("recoverable final reply"),
}),
});
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toMatchObject({
...pending,
deliveries: [{ id: "delivery-1", state: preserve ? "prepared" : "unknown" }],
});
});
it("clears pending final delivery after intentional pre-delivery cancellation", async () => {
hookMocks.runner.hasHooks.mockReturnValue(false);
sessionStoreMocks.currentEntry = {
sessionId: "session-1",
sessionKey: "agent:test:session",
pendingFinalDelivery: pendingFinalDelivery("policy-suppressed reply"),
};
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
existing: sessionStoreMocks.currentEntry,
});
const deliver = vi.fn().mockResolvedValue(undefined);
const dispatcher = createReplyDispatcher({
deliver,
beforeDeliver: () => null,
});
const result = await dispatchReplyFromConfig({
ctx: createHookCtx(),
cfg: emptyConfig,
dispatcher,
replyResolver: async () => pendingFinalReply("policy-suppressed reply"),
});
await dispatcher.waitForIdle();
await vi.waitFor(() => {
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined();
});
expect(result.queuedFinal).toBe(true);
expect(deliver).not.toHaveBeenCalled();
// createHookCtx's "private" chat type is undirected, so the cancelled final
// does not trigger a fallback attempt.
expect(dispatcher.getCancelledCounts?.()).toEqual({ tool: 0, block: 0, final: 1 });
expect(dispatcher.getFailedCounts?.()).toEqual({ tool: 0, block: 0, final: 0 });
expect(sessionStoreMocks.updateSessionEntry).toHaveBeenCalledTimes(2);
});
it("delivers a generated final reply before queued follow-up admission", async () => {
hookMocks.runner.hasHooks.mockReturnValue(false);
const dispatcher = createDispatcher();
@@ -4,10 +4,10 @@ import {
INTERNAL_RUNTIME_CONTEXT_BEGIN,
INTERNAL_RUNTIME_CONTEXT_END,
} from "../../agents/internal-runtime-context.js";
import { setReplyPayloadMetadata } from "../reply-payload.js";
import { markInboundContextLabel } from "./inbound-context-marker.js";
import {
buildRecoverablePendingFinalDeliveryText,
buildPendingFinalDeliveryText,
normalizePendingFinalDeliveryPayloads,
normalizePendingFinalRecoveryPayloads,
sanitizePendingFinalDeliveryText,
@@ -95,12 +95,12 @@ describe("normalizePendingFinalRecoveryPayloads", () => {
const rawPayloads = [{ text: "Rendered chart\nMEDIA:/tmp/chart.png" }];
const recoveryPayloads = normalizePendingFinalRecoveryPayloads(rawPayloads);
expect(buildPendingFinalDeliveryText(recoveryPayloads)).toBe(
expect(recoveryPayloads.map((payload) => payload.text)).toEqual([
"Rendered chart\nMEDIA:/tmp/chart.png",
);
]);
const deliveryPayloads = normalizePendingFinalDeliveryPayloads(rawPayloads);
expect(buildPendingFinalDeliveryText(deliveryPayloads)).toBe("Rendered chart");
expect(deliveryPayloads.map((payload) => payload.text)).toEqual(["Rendered chart"]);
});
it("keeps media-only directives as durable recovery text", () => {
@@ -108,7 +108,7 @@ describe("normalizePendingFinalRecoveryPayloads", () => {
{ text: "MEDIA:/tmp/chart.png" },
]);
expect(buildPendingFinalDeliveryText(recoveryPayloads)).toBe("MEDIA:/tmp/chart.png");
expect(recoveryPayloads.map((payload) => payload.text)).toEqual(["MEDIA:/tmp/chart.png"]);
expect(normalizePendingFinalDeliveryPayloads(recoveryPayloads)).toHaveLength(1);
});
@@ -139,6 +139,18 @@ describe("normalizePendingFinalRecoveryPayloads", () => {
).toBeUndefined();
});
it("separates implicit delivery threading from explicit reply semantics", () => {
expect(
buildRecoverablePendingFinalDeliveryText([
{ text: "Visible final", replyToId: "source-message" },
]),
).toBe("Visible final");
const explicitReply = { text: "Visible final", replyToId: "source-message" };
setReplyPayloadMetadata(explicitReply, { replyToIdExplicit: true });
expect(buildRecoverablePendingFinalDeliveryText([explicitReply])).toBeUndefined();
});
it("refuses multi-payload media finals because text recovery loses payload boundaries", () => {
expect(
buildRecoverablePendingFinalDeliveryText([
+20 -6
View File
@@ -1,5 +1,7 @@
import type { SessionEntry } from "../../config/sessions/types.js";
import type { DurableDeliveryCompletion } from "../../infra/outbound/delivery-completion.js";
import { normalizeReplyPayloadsForDelivery } from "../../infra/outbound/payloads.js";
import { getReplyPayloadMetadata, type ReplyPayload } from "../reply-payload.js";
import {
isSilentReplyPayloadText,
isSilentReplyText,
@@ -8,7 +10,6 @@ import {
stripLeadingSilentToken,
stripSilentToken,
} from "../tokens.js";
import type { ReplyPayload } from "../types.js";
import { stripInternalMetadataForDisplay } from "./display-text-sanitize.js";
import { normalizeReplyPayload } from "./normalize-reply.js";
@@ -38,12 +39,16 @@ export function buildRecoverablePendingFinalDeliveryText(
if (payload.isReasoning === true) {
continue;
}
const deliveryPayloads = normalizeReplyPayloadsForDelivery([payload]);
const recoveryPayload =
payload.replyToId && getReplyPayloadMetadata(payload)?.replyToIdExplicit !== true
? { ...payload, replyToId: undefined }
: payload;
const deliveryPayloads = normalizeReplyPayloadsForDelivery([recoveryPayload]);
if (deliveryPayloads.length === 0) {
continue;
}
if (
hasUnsupportedDurableRecoveryShape(payload) ||
hasUnsupportedDurableRecoveryShape(recoveryPayload) ||
deliveryPayloads.some(hasUnrecoverableNormalizedDeliveryShape)
) {
return undefined;
@@ -78,7 +83,7 @@ export function buildRecoverablePendingFinalDeliveryText(
}
/** Build the restart-recovery text represented by one or more final payloads. */
export function buildPendingFinalDeliveryText(payloads: ReplyPayload[]): string {
function buildPendingFinalDeliveryText(payloads: ReplyPayload[]): string {
const text = payloads
.filter((payload) => payload.isReasoning !== true)
.map((payload) => payload.text)
@@ -93,6 +98,15 @@ export const PENDING_FINAL_DELIVERY_CLEAR_PATCH = {
pendingFinalDelivery: undefined,
} as const satisfies Partial<SessionEntry>;
export function resolvePendingFinalDeliveryCompletion(
payloads: readonly ReplyPayload[] | undefined,
): Extract<DurableDeliveryCompletion, { kind: "pending-final" }> | undefined {
const completion = payloads
?.map((payload) => getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion)
.find(Boolean);
return completion ? { kind: "pending-final", ...completion } : undefined;
}
function collectDurableMediaDirectives(payload: ReplyPayload): string[] {
if (payload.sensitiveMedia === true) {
return [];
@@ -122,8 +136,8 @@ function hasUnsupportedDurableRecoveryShape(payload: ReplyPayload): boolean {
payload.channelData !== undefined ||
payload.location !== undefined ||
payload.replyToId !== undefined ||
payload.replyToTag !== undefined ||
payload.replyToCurrent !== undefined ||
payload.replyToTag === true ||
payload.replyToCurrent === true ||
payload.audioAsVoice === true ||
payload.videoAsNote === true ||
payload.spokenText !== undefined ||
+44 -3
View File
@@ -8,6 +8,7 @@ import {
isProvenDeliveryNotSentError,
} from "../../infra/delivery-recovery.shared.js";
import { collectErrorGraphCandidates } from "../../infra/errors.js";
import { settlePendingFinalDelivery } from "../../infra/outbound/delivery-completion.js";
import { generateSecureInt } from "../../infra/secure-random.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import type { SilentReplyConversationType } from "../../shared/silent-reply-policy.js";
@@ -442,6 +443,7 @@ export function createReplyDispatcher(options: ReplyDispatcherOptions): ReplyDis
): Promise<ReplyDispatchDeliveryOutcome> => {
let deliverPayload: ReplyPayload | null = payload;
let deliveryStarted = false;
const custody = getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion;
try {
if (beforeDeliver) {
try {
@@ -451,21 +453,60 @@ export function createReplyDispatcher(options: ReplyDispatcherOptions): ReplyDis
throw error;
}
if (!deliverPayload) {
// Record the intentional non-delivery before observers run so a
// restart during observer work cannot replay a suppressed final.
if (custody) {
await settlePendingFinalDelivery({ kind: "pending-final", ...custody }, "suppressed", [
"prepared",
]);
}
await notifyBeforeDeliverCancelled(payload, info);
return "cancelled";
}
deliverPayload = copyReplyPayloadMetadata(payload, deliverPayload);
}
if (custody) {
// Claim direct-send custody before provider I/O; a non-prepared marker
// means another owner already delivered, suppressed, or superseded this
// final, so repeating the send would duplicate it.
const claim = await settlePendingFinalDelivery(
{ kind: "pending-final", ...custody },
"queued",
["prepared"],
);
if (claim.state !== "queued") {
await notifyBeforeDeliverCancelled(payload, info);
return "cancelled";
}
}
deliveryStarted = true;
await options.deliver(deliverPayload, info);
if (custody) {
await settlePendingFinalDelivery({ kind: "pending-final", ...custody }, "delivered", [
"queued",
]);
}
return "delivered";
} catch (error) {
const outcome =
deliveryStarted && !isRetryableNoSendFailure(error)
? "failed-deliver"
: "failed-before-deliver";
if (custody && deliveryStarted) {
// Proven no-send keeps the marker replayable for restart recovery —
// including after direct custody escalated queued→unknown pre-I/O,
// since the error proves the send never crossed the wire. Anything
// else after platform I/O started fails closed as "unknown".
await settlePendingFinalDelivery(
{ kind: "pending-final", ...custody },
outcome === "failed-deliver" ? "unknown" : "prepared",
outcome === "failed-deliver" ? ["queued"] : ["queued", "unknown"],
);
}
try {
await options.onError?.(error, info);
} catch {}
return deliveryStarted && !isRetryableNoSendFailure(error)
? "failed-deliver"
: "failed-before-deliver";
return outcome;
}
};
@@ -13,6 +13,10 @@ export type ReplyFollowupAdmissionBarrierTimeoutPolicy = {
export type ReplyDispatchRuntimeInfo = {
kind: ReplyDispatchKind;
assistantMessageIndex?: number;
/** @internal Claim direct-send custody immediately before recipient-visible platform I/O. */
onPlatformSendDispatch?: () => Promise<void>;
/** @internal Bind this delivery's host-owned completion to a transformed payload. */
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
};
export type ReplyDispatchBeforeDeliver = (
@@ -29,6 +29,7 @@ type ReplyRestartRecoveryClaimController = {
state: Exclude<RestartRecoveryBeforeAgentReplyState, "admitted" | "pending">;
pendingFinalDelivery?: {
context?: DeliveryContext;
deliveries: NonNullable<SessionEntry["pendingFinalDelivery"]>["deliveries"];
intentId: string;
text: string;
};
@@ -376,6 +377,7 @@ export function createReplyRestartRecoveryClaimController(params: {
...(pendingFinalDelivery.intentId
? { intentId: pendingFinalDelivery.intentId }
: {}),
deliveries: pendingFinalDelivery.deliveries,
...(pendingFinalDelivery.context
? { context: pendingFinalDelivery.context }
: {}),
+24 -10
View File
@@ -4,6 +4,7 @@
* Sends rendered reply payloads, records live preview state, and classifies delivery outcomes.
*/
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
import { resolvePendingFinalDeliveryCompletion } from "../../auto-reply/reply/pending-final-delivery.js";
import { formatErrorMessage } from "../../infra/errors.js";
import type { OutboundDeliveryResult } from "../../infra/outbound/deliver-types.js";
import {
@@ -395,14 +396,27 @@ export async function withDurableMessageSendContext<T>(
export async function sendDurableMessageBatch(
params: DurableMessageSendContextParams,
): Promise<DurableMessageBatchSendResult> {
return await withDurableMessageSendContext(params, async (ctx) => {
const rendered = await ctx.render();
const result = await ctx.send(rendered);
if (result.status === "sent" || result.status === "suppressed") {
await ctx.commit(result.receipt);
} else {
await ctx.fail(result.error);
}
return result;
});
const pendingFinalCompletion = params.deliveryCompletion
? undefined
: resolvePendingFinalDeliveryCompletion(params.payloads);
const pendingFinalDelivery = pendingFinalCompletion
? {
deliveryCompletion: pendingFinalCompletion,
deliveryIntentId: pendingFinalCompletion.deliveryId,
durability: "required" as const,
}
: {};
return await withDurableMessageSendContext(
{ ...params, ...pendingFinalDelivery },
async (ctx) => {
const rendered = await ctx.render();
const result = await ctx.send(rendered);
if (result.status === "sent" || result.status === "suppressed") {
await ctx.commit(result.receipt);
} else {
await ctx.fail(result.error);
}
return result;
},
);
}
+2
View File
@@ -832,6 +832,8 @@ export type ChannelPollContext = {
silent?: boolean;
isAnonymous?: boolean;
gatewayClientScopes?: readonly string[];
/** @internal Refresh durable timing before recipient-visible platform I/O. */
onPlatformSendDispatch?: () => Promise<void>;
};
/** Minimal base for all channel probe results. Channel-specific probes extend this. */
@@ -0,0 +1,60 @@
import {
getReplyPayloadMetadata,
setReplyPayloadMetadata,
type ReplyPayload,
} from "../../auto-reply/reply-payload.js";
import { PlatformMessageNotDispatchedError } from "../../infra/outbound/deliver-types.js";
import { settlePendingFinalDelivery } from "../../infra/outbound/delivery-completion.js";
import type { ChannelDeliveryInfo } from "./types.js";
type DirectPendingFinalCustody = Pick<ChannelDeliveryInfo, "bindPendingFinalDelivery"> & {
onPlatformSendDispatch: () => Promise<void>;
};
export const NO_PENDING_FINAL_CUSTODY: DirectPendingFinalCustody = {
onPlatformSendDispatch: () => Promise.resolve(),
};
export function resolvePendingFinalCompletion(payload: ReplyPayload) {
const identity = getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion;
return identity ? { kind: "pending-final" as const, ...identity } : undefined;
}
export function createDirectPendingFinalCustody(
payload: ReplyPayload,
): DirectPendingFinalCustody | undefined {
const completion = resolvePendingFinalCompletion(payload);
if (!completion) {
return undefined;
}
const { kind: _kind, ...identity } = completion;
let admission: Promise<void> | undefined;
return {
bindPendingFinalDelivery: (nextPayload) =>
setReplyPayloadMetadata(nextPayload, {
pendingFinalDeliveryCompletion: identity,
}),
onPlatformSendDispatch: () => {
admission ??= settlePendingFinalDelivery(completion, "unknown", ["prepared", "queued"]).then(
(result) => {
if (result.state !== "unknown") {
throw new PlatformMessageNotDispatchedError(
"Pending final delivery ownership changed before platform dispatch",
{ cause: new Error(`pending final delivery is ${result.state}`) },
);
}
},
);
return admission;
},
};
}
export function toCoreManagedDeliveryInfo(info: ChannelDeliveryInfo) {
return {
kind: info.kind,
...(info.assistantMessageIndex === undefined
? {}
: { assistantMessageIndex: info.assistantMessageIndex }),
};
}
+3 -2
View File
@@ -204,7 +204,6 @@ export async function deliverInboundReplyWithMessageSendContext(
requesterSenderUsername: params.ctxPayload.SenderUsername,
requesterSenderE164: params.ctxPayload.SenderE164,
});
const send = await sendDurableMessageBatch({
cfg: params.cfg,
channel,
@@ -220,7 +219,9 @@ export async function deliverInboundReplyWithMessageSendContext(
mediaAccess: params.mediaAccess,
silent: params.silent,
durability,
...(durability === "required" ? { requireUnknownSendReconciliation: true } : {}),
...(requiredCapabilities.reconcileUnknownSend === true
? { requireUnknownSendReconciliation: true }
: {}),
session,
gatewayClientScopes: params.ctxPayload.GatewayClientScopes ?? [],
});
+58 -19
View File
@@ -1,5 +1,6 @@
import { dispatchInboundMessageWithRoutedChannelDispatcher } from "../../auto-reply/dispatch.js";
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
import { copyReplyPayloadMetadata, type ReplyPayload } from "../../auto-reply/reply-payload.js";
import { suppressPendingFinalDelivery } from "../../auto-reply/reply/dispatch-from-config.pending-final.js";
import type { DispatchFromConfigResult } from "../../auto-reply/reply/dispatch-from-config.types.js";
import type { ReplyDispatchKind } from "../../auto-reply/reply/reply-dispatcher.types.js";
import { runWithSessionInitConflictRetry } from "../../auto-reply/reply/session-init-conflict-retry.js";
@@ -12,6 +13,7 @@ import { formatErrorMessage, toErrorObject } from "../../infra/errors.js";
import { applyMessageSendingHook } from "../../infra/outbound/deliver-hooks.js";
import { normalizeEmptyPayloadForDelivery } from "../../infra/outbound/deliver-payload.js";
import { isPlatformMessageNotDispatchedError } from "../../infra/outbound/deliver-types.js";
import { settlePendingFinalDelivery } from "../../infra/outbound/delivery-completion.js";
import { createMessageSentEmitter } from "../../infra/outbound/message-sent-hook.js";
import { summarizeOutboundPayloadForTransport } from "../../infra/outbound/payloads.js";
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
@@ -19,6 +21,12 @@ import { resolveMessageReceiptPrimaryId } from "../message/receipt.js";
import { createChannelReplyPipeline } from "../message/reply-pipeline.js";
import { recordInboundSession } from "../session.js";
import { isChannelPartialDeliveryError } from "./delivery-result.js";
import {
createDirectPendingFinalCustody,
NO_PENDING_FINAL_CUSTODY,
resolvePendingFinalCompletion,
toCoreManagedDeliveryInfo,
} from "./direct-delivery-custody.js";
import {
deliverInboundReplyWithMessageSendContext,
isDurableInboundReplyDeliveryHandled,
@@ -266,6 +274,13 @@ async function settleChannelDeliveryAttempt(params: {
messageId: resolveChannelDeliveryMessageId(finalized),
});
}
const completion = resolvePendingFinalCompletion(attempt.payload);
if (completion) {
await settlePendingFinalDelivery(
completion,
isExplicitlyNonVisibleChannelDelivery(finalized) ? "suppressed" : "delivered",
);
}
await runChannelDeliveryObserver({
onDelivered: params.onDelivered,
payload: attempt.payload,
@@ -332,7 +347,7 @@ async function applyRoutedDirectMessageSending(params: {
}),
};
}
return { payload };
return { payload: copyReplyPayloadMetadata(params.payload, payload) };
}
function reconcileNonVisibleChannelDeliveries(
@@ -465,6 +480,7 @@ async function dispatchChannelTurnWithDeliveryOwner(
| "cancelled_by_reply_payload_sending_hook"
| "empty_after_reply_payload_sending_hook",
) => {
await suppressPendingFinalDelivery(payload);
await runChannelDeliveryObserver({
onDelivered: delivery.onDelivered,
payload,
@@ -477,13 +493,18 @@ async function dispatchChannelTurnWithDeliveryOwner(
dispatcherOptions: {
...replyPipeline.dispatcherOptions,
deliver: async (payload: ReplyPayload, info: ChannelDeliveryInfo) => {
const preparedPayload = delivery.preparePayload
const preparedPayloadResult = delivery.preparePayload
? await delivery.preparePayload(payload, info)
: payload;
const preparedPayload =
preparedPayloadResult === null
? null
: copyReplyPayloadMetadata(payload, preparedPayloadResult);
if (preparedPayload === null) {
const suppression = createSuppressedChannelDeliveryResult({
reason: "no_visible_payload",
});
await suppressPendingFinalDelivery(payload);
await runChannelDeliveryObserver({
onDelivered: delivery.onDelivered,
payload,
@@ -525,15 +546,22 @@ async function dispatchChannelTurnWithDeliveryOwner(
}
let effectivePayload = preparedPayload;
let result: ChannelDeliveryResult | void = undefined;
let directInfo: ChannelDeliveryInfo = info;
try {
if (
ownership === "routed-delivery" &&
"deliverWithProviderMessageSending" in delivery &&
delivery.deliverWithProviderMessageSending
) {
const providerInfo = {
...info,
...(createDirectPendingFinalCustody(effectivePayload) ??
NO_PENDING_FINAL_CUSTODY),
};
directInfo = providerInfo;
result = await delivery.deliverWithProviderMessageSending(
effectivePayload,
info,
providerInfo,
);
} else {
if (
@@ -555,13 +583,22 @@ async function dispatchChannelTurnWithDeliveryOwner(
"channel delivery adapter is missing a direct deliverer",
);
}
result = await delivery.deliver(effectivePayload, info);
const custody = createDirectPendingFinalCustody(effectivePayload);
await custody?.onPlatformSendDispatch();
result = await delivery.deliver(
effectivePayload,
toCoreManagedDeliveryInfo(info),
);
}
}
} catch (error: unknown) {
if (delivery.observeMessageSent) {
await settleChannelDeliveryAttempt({
attempt: { payload: effectivePayload, info, error },
attempt: {
payload: effectivePayload,
info: directInfo,
error,
},
onDelivered: delivery.onDelivered,
emitMessageSent: getMessageSentEmitter()?.emitMessageSent,
});
@@ -572,22 +609,24 @@ async function dispatchChannelTurnWithDeliveryOwner(
// Finalization can reject while the buffered dispatcher is still unwinding.
// Observe it now; settlement still awaits the original promise and its error.
void result.finalization.catch(() => undefined);
pendingDeliveryAttempts.push({ payload: effectivePayload, info, result });
} else if (delivery.observeMessageSent) {
const finalized = await settleChannelDeliveryAttempt({
attempt: { payload: effectivePayload, info, result },
onDelivered: delivery.onDelivered,
emitMessageSent: getMessageSentEmitter()?.emitMessageSent,
});
recordSettledDelivery(info, finalized);
} else {
await runChannelDeliveryObserver({
onDelivered: delivery.onDelivered,
pendingDeliveryAttempts.push({
payload: effectivePayload,
info,
info: directInfo,
result,
});
recordSettledDelivery(info, result ?? undefined);
} else {
const finalized = await settleChannelDeliveryAttempt({
attempt: {
payload: effectivePayload,
info: directInfo,
result,
},
onDelivered: delivery.onDelivered,
emitMessageSent: delivery.observeMessageSent
? getMessageSentEmitter()?.emitMessageSent
: undefined,
});
recordSettledDelivery(info, finalized);
}
return result;
},
@@ -1,6 +1,10 @@
// Channel turn delivery tests cover orchestration, dispatch, and completion behavior.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
import {
getReplyPayloadMetadata,
setReplyPayloadMetadata,
type ReplyPayload,
} from "../../auto-reply/reply-payload.js";
import type { DispatchReplyWithBufferedBlockDispatcher } from "../../auto-reply/reply/provider-dispatcher.types.js";
import type { FinalizedMsgContext } from "../../auto-reply/templating.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
@@ -10,7 +14,7 @@ import { outboundMessageIdentities } from "../message/outbound-echo-state.js";
import type { RecordInboundSession } from "../session.types.js";
import { hasVisibleChannelTurnDispatch } from "./dispatch-result.js";
import { dispatchAssembledChannelTurn, dispatchRoutedChannelTurn } from "./lifecycle.js";
import type { ChannelTurnResult } from "./types.js";
import type { ChannelDeliveryInfo, ChannelTurnResult } from "./types.js";
const deliverOutboundPayloads = vi.hoisted(() => vi.fn());
const resolveOutboundDurableFinalDeliverySupport = vi.hoisted(() => vi.fn());
@@ -24,6 +28,9 @@ const createMessageSentEmitter = vi.hoisted(() =>
vi.fn(() => ({ emitMessageSent, hasMessageSentHooks: true })),
);
const readRecentUserAssistantTextForSession = vi.hoisted(() => vi.fn());
const settlePendingFinalDelivery = vi.hoisted(() =>
vi.fn(async (_completion: unknown, state: string) => ({ state })),
);
vi.mock("../../auto-reply/reply/provider-dispatcher.js", async (importOriginal) => {
const actual =
@@ -77,6 +84,12 @@ vi.mock("../../config/sessions/transcript.js", () => ({
readRecentUserAssistantTextForSession,
}));
vi.mock("../../infra/outbound/delivery-completion.js", async (importOriginal) => {
const actual =
await importOriginal<typeof import("../../infra/outbound/delivery-completion.js")>();
return { ...actual, settlePendingFinalDelivery };
});
const cfg = {} as OpenClawConfig;
function createCtx(overrides: Partial<FinalizedMsgContext> = {}): FinalizedMsgContext {
@@ -286,6 +299,77 @@ describe("channel turn delivery", () => {
expect(result.dispatchResult.counts.final).toBe(1);
});
it("preserves pending final custody through preparation and message hook rewrites", async () => {
const order: string[] = [];
const completion = {
deliveryId: "delivery-1",
intentId: "intent-1",
sessionId: "session-1",
sessionKey: "agent:main:telegram:peer",
storePath: "/tmp/sessions.json",
};
const sourcePayload = setReplyPayloadMetadata(
{ text: "reply" },
{ pendingFinalDeliveryCompletion: completion },
);
dispatchReplyWithRoutedChannelDispatcherCore.mockImplementationOnce(async (params) => {
await params.dispatcherOptions.deliver(sourcePayload, { kind: "final" });
return { queuedFinal: true, counts: { tool: 0, block: 0, final: 1 } };
});
getGlobalHookRunner.mockReturnValue({
hasHooks: (name: string) => name === "message_sending",
runMessageSending: vi.fn(async ({ content }: { content: string }) => ({
content: `${content} + hook`,
})),
});
let releaseDelivery: (() => void) | undefined;
const deliveryPending = new Promise<void>((resolve) => {
releaseDelivery = resolve;
});
settlePendingFinalDelivery.mockImplementationOnce(async (_completion, state: string) => {
order.push(`settle:${state}`);
return { state };
});
const deliver = vi.fn(async (payload: ReplyPayload, info: ChannelDeliveryInfo) => {
expect(getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion).toEqual(completion);
expect("onPlatformSendDispatch" in info).toBe(false);
order.push("signal:accepted");
await deliveryPending;
return { messageIds: ["direct-1"], visibleReplySent: true };
});
const dispatch = dispatchRoutedChannelTurn({
cfg,
channel: "telegram",
accountId: "acct",
route: { agentId: "main", sessionKey: completion.sessionKey },
ctxPayload: createCtx({ Surface: "telegram", OriginatingTo: "chat-1" }),
delivery: {
preparePayload: (payload) => ({ ...payload, text: `${payload.text} + prepared` }),
deliver,
},
});
await vi.waitFor(() => expect(deliver).toHaveBeenCalledOnce());
expect(order).toEqual(["settle:unknown", "signal:accepted"]);
releaseDelivery?.();
await dispatch;
expect(deliver).toHaveBeenCalledOnce();
expect(deliver.mock.calls[0]?.[0]).toMatchObject({ text: "reply + prepared + hook" });
expect(settlePendingFinalDelivery).toHaveBeenNthCalledWith(
1,
{ kind: "pending-final", ...completion },
"unknown",
["prepared", "queued"],
);
expect(settlePendingFinalDelivery).toHaveBeenNthCalledWith(
2,
{ kind: "pending-final", ...completion },
"delivered",
);
});
it("does not let message hooks resurrect payloads suppressed during preparation", async () => {
const runMessageSending = vi.fn(async () => ({ content: "resurrected" }));
getGlobalHookRunner.mockReturnValue({
@@ -454,7 +538,10 @@ describe("channel turn delivery", () => {
expect(deliverWithProviderMessageSending).toHaveBeenCalledWith(
{ text: "reply" },
{ kind: "final" },
expect.objectContaining({
kind: "final",
onPlatformSendDispatch: expect.any(Function),
}),
);
expect(runMessageSending).not.toHaveBeenCalled();
});
+12 -5
View File
@@ -9,7 +9,7 @@ import type { GetReplyFromConfig } from "../../auto-reply/reply/get-reply.types.
import type { HistoryEntry, HistoryMediaEntry } from "../../auto-reply/reply/history.types.js";
import type { DispatchReplyWithBufferedBlockDispatcher } from "../../auto-reply/reply/provider-dispatcher.types.js";
import type { ReplyDispatcherWithTypingOptions } from "../../auto-reply/reply/reply-dispatcher.js";
import type { ReplyDispatchKind } from "../../auto-reply/reply/reply-dispatcher.types.js";
import type { ReplyDispatchRuntimeInfo } from "../../auto-reply/reply/reply-dispatcher.types.js";
import type {
FinalizedMsgContext,
InboundSourceModality,
@@ -156,8 +156,15 @@ export type PreflightFacts = {
};
/** Delivery metadata for one reply payload dispatch. */
export type ChannelDeliveryInfo = {
kind: ReplyDispatchKind;
export type ChannelDeliveryInfo = ReplyDispatchRuntimeInfo;
type ChannelCoreManagedDeliveryInfo = Omit<
ChannelDeliveryInfo,
"bindPendingFinalDelivery" | "onPlatformSendDispatch"
>;
type ChannelProviderOwnedDeliveryInfo = ChannelDeliveryInfo & {
onPlatformSendDispatch: () => Promise<void>;
};
/** Durable delivery queue intent recorded when a reply is deferred. */
@@ -220,7 +227,7 @@ type ChannelDeliveryAdapterBase = {
export type ChannelCoreManagedTurnDeliveryAdapter = ChannelDeliveryAdapterBase & {
deliver: (
payload: ReplyPayload,
info: ChannelDeliveryInfo,
info: ChannelCoreManagedDeliveryInfo,
) => Promise<ChannelDeliveryResult | void>;
durable?:
| false
@@ -245,7 +252,7 @@ export type ChannelProviderOwnedMessageSendingDeliveryAdapter = ChannelDeliveryA
*/
deliverWithProviderMessageSending: (
payload: ReplyPayload,
info: ChannelDeliveryInfo,
info: ChannelProviderOwnedDeliveryInfo,
) => Promise<ChannelDeliveryResult | void>;
deliver?: never;
durable?: never;
+1
View File
@@ -85,6 +85,7 @@ vi.mock("../agents/command/session-store.runtime.js", async () => {
const accessor = await import("../config/sessions/session-accessor.js");
return {
loadSessionEntry: accessor.loadSessionEntry,
loadSessionEntryReadOnly: accessor.loadSessionEntryReadOnly,
updateSessionStoreAfterAgentRun: vi.fn(async () => undefined),
};
});
+29
View File
@@ -81,6 +81,35 @@ it("normalizes boolean-only pending delivery as transport-only", () => {
});
});
it("normalizes exact pending-final delivery owners", () => {
expect(
normalizePersistedSessionEntryShape({
sessionId: "session-1",
updatedAt: 42,
pendingFinalDelivery: {
kind: "replayable",
text: "durable reply",
createdAt: 41,
intentId: "intent-1",
deliveries: [
{ id: "delivery-prepared", state: "prepared" },
{ id: "delivery-delivered", state: "delivered" },
{ id: "", state: "queued" },
{ id: "delivery-invalid", state: "invalid" },
],
},
}),
).toMatchObject({
pendingFinalDelivery: {
intentId: "intent-1",
deliveries: [
{ id: "delivery-prepared", state: "prepared" },
{ id: "delivery-delivered", state: "delivered" },
],
},
});
});
it("normalizes and preserves the durable assistant transcript repair backlog", () => {
expect(
normalizePersistedSessionEntryShape({
+20
View File
@@ -144,10 +144,30 @@ function normalizePendingFinalDelivery(
return undefined;
}
const intentId = normalizeOptionalString(value.intentId);
const deliveries: NonNullable<SessionEntry["pendingFinalDelivery"]>["deliveries"] = Array.isArray(
value.deliveries,
)
? value.deliveries.flatMap((delivery) => {
if (!isRecord(delivery)) {
return [];
}
const id = normalizeOptionalString(delivery.id);
const state = delivery.state;
return id &&
(state === "prepared" ||
state === "queued" ||
state === "delivered" ||
state === "suppressed" ||
state === "unknown")
? [{ id, state }]
: [];
})
: undefined;
const base = {
createdAt,
...(isRecord(value.context) ? { context: value.context } : {}),
...(intentId ? { intentId } : {}),
...(deliveries ? { deliveries } : {}),
};
if (value.kind === "transport-only") {
return { kind: "transport-only", ...base };
+4
View File
@@ -66,6 +66,10 @@ type PendingFinalDeliveryState = {
createdAt: number;
context?: DeliveryContext;
intentId?: string;
deliveries?: Array<{
id: string;
state: "prepared" | "queued" | "delivered" | "suppressed" | "unknown";
}>;
} & ({ kind: "replayable"; text: string } | { kind: "transport-only" });
/**
+17 -5
View File
@@ -303,9 +303,13 @@ export async function deliverOutboundPayloadsWithQueueCleanup(
if (!queueId) {
if (params.deliveryCompletion) {
if (results.length > 0) {
completeDurableDelivery(params.deliveryCompletion, results.at(-1)!);
await completeDurableDelivery(
params.deliveryCompletion,
results.at(-1)!,
platformQueueStateDir,
);
} else {
suppressDurableDelivery(params.deliveryCompletion);
await suppressDurableDelivery(params.deliveryCompletion, platformQueueStateDir);
}
}
if (!params.deferCommitHooks) {
@@ -363,9 +367,13 @@ export async function deliverOutboundPayloadsWithQueueCleanup(
} else {
if (params.deliveryCompletion) {
if (results.length > 0) {
completeDurableDelivery(params.deliveryCompletion, results.at(-1)!);
await completeDurableDelivery(
params.deliveryCompletion,
results.at(-1)!,
platformQueueStateDir,
);
} else {
suppressDurableDelivery(params.deliveryCompletion);
await suppressDurableDelivery(params.deliveryCompletion, platformQueueStateDir);
}
}
const postSendState =
@@ -562,7 +570,11 @@ export async function deliverOutboundPayloadsWithQueueCleanup(
terminalRejectionHandled = true;
} else {
if (params.deliveryCompletion) {
rejectDurableDelivery(params.deliveryCompletion, permanentRejection.message);
await rejectDurableDelivery(
params.deliveryCompletion,
permanentRejection.message,
platformQueueStateDir,
);
ownerRejected = true;
}
await (producerClaimId
+16
View File
@@ -14,7 +14,9 @@ import {
stageAndEnqueueOutboundDelivery,
} from "./deliver-queue-admission.js";
import { deliverOutboundPayloadsWithQueueCleanup } from "./deliver-queue-execute.js";
import { createQueuedDeliveryOwner } from "./deliver-queue-state.js";
import type { OutboundDeliveryResult } from "./deliver-types.js";
import { markDurableDeliveryQueued } from "./delivery-completion.js";
import { startDeliveryProducerLease } from "./delivery-queue-lease.js";
import {
StableDeliveryPreparationLostError,
@@ -272,6 +274,20 @@ async function runOutboundDeliveryWithQueue(
if (queued?.created && stablePreparationOwner) {
stablePreparationOwner.markPublished();
}
if (queueId && params.deliveryCompletion) {
const completion = await markDurableDeliveryQueued(
params.deliveryCompletion,
queueId,
queued?.created ? "prepared" : undefined,
);
if (completion.state !== "queued") {
await createQueuedDeliveryOwner({
queueId,
expectedPlatformSendAttemptId: queued?.producerClaimId,
}).ack({ suppressCompletionReceipt: true });
return [];
}
}
if (queueId) {
params.onDeliveryIntent?.({
id: queueId,
+26
View File
@@ -96,6 +96,7 @@ const queueMocks = vi.hoisted(() => ({
}));
const completionMocks = vi.hoisted(() => ({
completeDurableDelivery: vi.fn(),
markDurableDeliveryQueued: vi.fn(async () => ({ state: "queued" as const })),
rejectDurableDelivery: vi.fn(),
suppressDurableDelivery: vi.fn(),
}));
@@ -207,6 +208,7 @@ vi.mock("./delivery-queue.js", () => ({
}));
vi.mock("./delivery-completion.js", () => ({
completeDurableDelivery: completionMocks.completeDurableDelivery,
markDurableDeliveryQueued: completionMocks.markDurableDeliveryQueued,
rejectDurableDelivery: completionMocks.rejectDurableDelivery,
suppressDurableDelivery: completionMocks.suppressDurableDelivery,
}));
@@ -521,6 +523,7 @@ describe("deliverOutboundPayloads", () => {
},
);
completionMocks.completeDurableDelivery.mockClear();
completionMocks.markDurableDeliveryQueued.mockClear();
completionMocks.rejectDurableDelivery.mockClear();
completionMocks.suppressDurableDelivery.mockClear();
queueMocks.ackDelivery.mockClear();
@@ -932,6 +935,26 @@ describe("deliverOutboundPayloads", () => {
expect(results[0]?.messageId).toBe("message-adapter-1");
});
it("does not claim platform custody when message adapter preflight fails", async () => {
const messageSendText = vi.fn();
setMatrixMessageAdapter({
id: "matrix",
durableFinal: { capabilities: { text: true } },
send: {
lifecycle: {
beforeSendAttempt: () => {
throw new Error("preflight rejected");
},
},
text: messageSendText,
},
});
await expect(deliverMatrix({ queuePolicy: "required" })).rejects.toThrow("preflight rejected");
expect(queueMocks.markDeliveryPlatformSendDispatched).not.toHaveBeenCalled();
expect(messageSendText).not.toHaveBeenCalled();
});
it("does not cross platform I/O when a stable queue intent already exists", async () => {
hookMocks.runner.hasHooks.mockImplementation((name?: string) => name === "message_sending");
queueMocks.findDeliveryIntentOwner.mockReturnValue({
@@ -1063,6 +1086,7 @@ describe("deliverOutboundPayloads", () => {
expect(completionMocks.completeDurableDelivery).toHaveBeenCalledWith(
expect.objectContaining({ operationId: "operation-chunked" }),
expect.objectContaining({ messageId: "chunk-2" }),
undefined,
);
});
@@ -2169,6 +2193,7 @@ describe("deliverOutboundPayloads", () => {
expect(completionMocks.rejectDurableDelivery).toHaveBeenCalledWith(
expect.objectContaining({ operationId: "operation-rejected" }),
"atomic message limit",
undefined,
);
expect(queueMocks.failDeliveryBeforePlatformSend).not.toHaveBeenCalled();
expect(queueMocks.failDelivery).not.toHaveBeenCalled();
@@ -2206,6 +2231,7 @@ describe("deliverOutboundPayloads", () => {
expect(completionMocks.rejectDurableDelivery).toHaveBeenCalledWith(
expect.objectContaining({ operationId: "operation-empty-rejection" }),
"Platform rejected the message before dispatch",
undefined,
);
});
@@ -0,0 +1,91 @@
import path from "node:path";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
import { createEmptyPluginRegistry } from "../../plugins/registry.js";
import { resetPluginRuntimeStateForTest, setActivePluginRegistry } from "../../plugins/runtime.js";
import { createOutboundTestPlugin, createTestRegistry } from "../../test-utils/channel-plugins.js";
import { matrixOutboundForQueueTest } from "./deliver.queue-integration.test-support.js";
import { loadPendingDeliveries } from "./delivery-queue-storage.js";
import { installDeliveryQueueTmpDirHooks } from "./delivery-queue.test-helpers.js";
let deliverOutboundPayloads: typeof import("./deliver.js").deliverOutboundPayloads;
describe("pending-final durable delivery completion", () => {
const fixtures = installDeliveryQueueTmpDirHooks();
let tmpDir: string;
beforeAll(async () => {
({ deliverOutboundPayloads } = await import("./deliver.js"));
});
beforeEach(() => {
tmpDir = fixtures.tmpDir();
setActivePluginRegistry(
createTestRegistry([
{
pluginId: "matrix",
source: "test",
plugin: createOutboundTestPlugin({ id: "matrix", outbound: matrixOutboundForQueueTest }),
},
]),
);
});
afterEach(() => {
resetPluginRuntimeStateForTest();
setActivePluginRegistry(createEmptyPluginRegistry());
});
it("suppresses a second stable caller after the exact pending final was delivered", async () => {
process.env.OPENCLAW_STATE_DIR = tmpDir;
const sessionKey = "agent:main:matrix:direct:123";
const storePath = path.join(tmpDir, "sessions.json");
const deliveryId = "pending-final-delivery-1";
const completion = {
kind: "pending-final" as const,
deliveryId,
intentId: "pending-final-intent-1",
sessionId: "session-1",
sessionKey,
storePath,
};
await replaceSessionEntry(
{ sessionKey, storePath },
{
sessionId: "session-1",
status: "running",
updatedAt: Date.now(),
pendingFinalDelivery: {
kind: "replayable",
text: "deliver once",
createdAt: Date.now(),
intentId: completion.intentId,
deliveries: [{ id: deliveryId, state: "prepared" }],
},
},
);
const sendMatrix = vi.fn().mockResolvedValue({ messageId: "matrix-message-1" });
const params = {
cfg: {} as OpenClawConfig,
channel: "matrix" as const,
to: "!room:example",
payloads: [{ text: "deliver once" }],
deps: { matrix: sendMatrix },
queuePolicy: "required" as const,
deliveryIntentId: deliveryId,
deliveryCompletion: completion,
};
await expect(deliverOutboundPayloads(params)).resolves.toMatchObject([
{ messageId: "matrix-message-1" },
]);
expect(loadSessionEntry({ sessionKey, storePath })?.pendingFinalDelivery?.deliveries).toEqual([
{ id: deliveryId, state: "delivered" },
]);
await expect(deliverOutboundPayloads(params)).resolves.toEqual([]);
expect(sendMatrix).toHaveBeenCalledOnce();
expect(await loadPendingDeliveries(tmpDir)).toEqual([]);
});
});
@@ -0,0 +1,130 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { commitMainSessionRecovery } from "../../agents/main-session-recovery/main-session-recovery-store.js";
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
import type { InternalSessionEntry } from "../../config/sessions/types.js";
import { settlePendingFinalDelivery } from "./delivery-completion.js";
const recoveryMocks = vi.hoisted(() => ({
scheduleMainSessionRecoveryPendingTarget: vi.fn(),
}));
vi.mock(
"../../agents/main-session-recovery/main-session-recovery-owner-release.js",
() => recoveryMocks,
);
describe("pending-final delivery completion", () => {
let tmpDir: string;
let storePath: string;
const sessionKey = "agent:main:main";
const completion = {
kind: "pending-final" as const,
deliveryId: "delivery-1",
intentId: "intent-1",
sessionId: "session-1",
sessionKey,
storePath: "",
};
beforeEach(async () => {
vi.clearAllMocks();
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-delivery-completion-"));
storePath = path.join(tmpDir, "sessions.json");
completion.storePath = storePath;
const entry: InternalSessionEntry = {
sessionId: completion.sessionId,
status: "running",
abortedLastRun: true,
updatedAt: Date.now(),
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 1,
},
pendingFinalDelivery: {
kind: "replayable",
text: "durable final",
createdAt: Date.now(),
intentId: completion.intentId,
deliveries: [{ id: completion.deliveryId, state: "prepared" }],
},
};
await replaceSessionEntry({ sessionKey, storePath }, entry);
});
afterEach(async () => {
await fs.rm(tmpDir, { recursive: true, force: true });
});
it("invalidates an earlier recovery decision and wakes the exact session", async () => {
const observation = { sessionId: completion.sessionId, cycleId: "cycle-1", revision: 1 };
await expect(settlePendingFinalDelivery(completion, "delivered")).resolves.toEqual({
state: "delivered",
});
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
mainRestartRecovery: { revision: 2 },
pendingFinalDelivery: {
deliveries: [{ id: completion.deliveryId, state: "delivered" }],
},
});
expect(recoveryMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith({
sessionId: completion.sessionId,
sessionKey,
storePath,
});
await expect(
commitMainSessionRecovery({
command: { kind: "fail_recovery", now: Date.now(), observation },
requireWriteSuccess: true,
target: { sessionKey, storePath },
}),
).resolves.toMatchObject({ transition: { kind: "rejected", reason: "stale_revision" } });
});
it("records queue custody without waking recovery", async () => {
await expect(settlePendingFinalDelivery(completion, "queued")).resolves.toEqual({
state: "queued",
});
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
mainRestartRecovery: { revision: 2 },
pendingFinalDelivery: {
deliveries: [{ id: completion.deliveryId, state: "queued" }],
},
});
expect(recoveryMocks.scheduleMainSessionRecoveryPendingTarget).not.toHaveBeenCalled();
});
it("carries the custom queue root when a terminal sibling wakes recovery", async () => {
const entry = loadSessionEntry({ sessionKey, storePath })!;
await replaceSessionEntry(
{ sessionKey, storePath },
{
...entry,
pendingFinalDelivery: {
...entry.pendingFinalDelivery!,
deliveries: [
{ id: completion.deliveryId, state: "prepared" },
{ id: "delivery-2", state: "queued" },
],
},
},
);
await expect(
settlePendingFinalDelivery(completion, "delivered", undefined, tmpDir),
).resolves.toEqual({ state: "delivered" });
expect(recoveryMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith({
sessionId: completion.sessionId,
sessionKey,
stateDir: tmpDir,
storePath,
});
});
});
+172 -33
View File
@@ -7,74 +7,213 @@ import {
markConversationDeliveryUnknown,
type ConversationDeliveryRecord,
} from "../../config/sessions/conversation-delivery-store.js";
import { updateSessionEntry } from "../../config/sessions/session-accessor.js";
import type { InternalSessionEntry } from "../../config/sessions/types.js";
import type { OutboundDeliveryResult } from "./deliver-types.js";
/** Serializable owner callback for a durable queue entry. */
export type DurableDeliveryCompletion = {
kind: "conversation";
agentId: string;
operationId: string;
storePath?: string;
export type DurableDeliveryCompletion =
| {
kind: "conversation";
agentId: string;
operationId: string;
storePath?: string;
}
| {
kind: "pending-final";
deliveryId: string;
intentId: string;
sessionId: string;
sessionKey: string;
storePath: string;
};
type DurableDeliveryCompletionResult = {
state: "prepared" | "queued" | "delivered" | "suppressed" | "rejected" | "unknown" | "stale";
platformMessageId?: string;
rejectionError?: string;
};
function scopeForCompletion(completion: DurableDeliveryCompletion) {
function scopeForCompletion(
completion: Extract<DurableDeliveryCompletion, { kind: "conversation" }>,
) {
return {
agentId: completion.agentId,
...(completion.storePath ? { storePath: completion.storePath } : {}),
};
}
function conversationResult(record: ConversationDeliveryRecord): DurableDeliveryCompletionResult {
const delivered = record.status === "sent" || record.status === "replied";
return {
state: delivered
? "delivered"
: record.status === "suppressed" ||
record.status === "rejected" ||
record.status === "unknown"
? record.status
: "queued",
...(delivered && (record.platformMessageId || record.preparedMessageId)
? { platformMessageId: record.platformMessageId ?? record.preparedMessageId }
: {}),
...(record.status === "rejected" && record.rejectionError
? { rejectionError: record.rejectionError }
: {}),
};
}
export async function settlePendingFinalDelivery(
completion: Extract<DurableDeliveryCompletion, { kind: "pending-final" }>,
state: Exclude<DurableDeliveryCompletionResult["state"], "rejected" | "stale">,
expectedStates?: readonly ("prepared" | "queued" | "unknown")[],
stateDir?: string,
): Promise<DurableDeliveryCompletionResult> {
let settled: DurableDeliveryCompletionResult["state"] = "stale";
let wakeRecovery = false;
await updateSessionEntry(
{ sessionKey: completion.sessionKey, storePath: completion.storePath },
(entry) => {
const internalEntry: InternalSessionEntry = entry;
if (
internalEntry.sessionId !== completion.sessionId ||
internalEntry.pendingFinalDelivery?.intentId !== completion.intentId
) {
return null;
}
const deliveries = internalEntry.pendingFinalDelivery.deliveries;
const index = deliveries?.findIndex(({ id }) => id === completion.deliveryId) ?? -1;
if (!deliveries || index < 0) {
return null;
}
const current = deliveries[index]!.state;
if (expectedStates && !expectedStates.some((expected) => expected === current)) {
return null;
}
const terminal =
current === "delivered" ||
current === "suppressed" ||
(current === "unknown" && state === "unknown");
settled = terminal ? current : state;
if (settled === current) {
return null;
}
wakeRecovery =
settled !== "queued" &&
internalEntry.status === "running" &&
internalEntry.abortedLastRun === true;
return {
...(internalEntry.mainRestartRecovery
? {
mainRestartRecovery: {
...internalEntry.mainRestartRecovery,
revision: internalEntry.mainRestartRecovery.revision + 1,
},
}
: {}),
pendingFinalDelivery: {
...internalEntry.pendingFinalDelivery,
deliveries: deliveries.with(index, { id: completion.deliveryId, state: settled }),
},
updatedAt: Date.now(),
};
},
{ skipMaintenance: true, takeCacheOwnership: true },
);
if (wakeRecovery) {
const { scheduleMainSessionRecoveryPendingTarget } =
await import("../../agents/main-session-recovery/main-session-recovery-owner-release.js");
scheduleMainSessionRecoveryPendingTarget({
sessionId: completion.sessionId,
sessionKey: completion.sessionKey,
...(stateDir !== undefined ? { stateDir } : {}),
storePath: completion.storePath,
});
}
return { state: settled };
}
function readPlatformMessageId(result: OutboundDeliveryResult): string | undefined {
const receiptId = result.receipt ? resolveMessageReceiptPrimaryId(result.receipt) : undefined;
return receiptId ?? (result.messageId.trim() || undefined);
}
/** Records queue ownership before either the live sender or recovery crosses platform I/O. */
export function markDurableDeliveryQueued(
export async function markDurableDeliveryQueued(
completion: DurableDeliveryCompletion,
queueId: string,
): ConversationDeliveryRecord {
return markConversationDeliveryQueued(
scopeForCompletion(completion),
completion.operationId,
queueId,
);
expectedPendingFinalState?: "prepared",
): Promise<DurableDeliveryCompletionResult> {
return completion.kind === "pending-final"
? // The reply dispatcher may have claimed direct custody ("queued") before the
// durable enqueue; both states still belong to this send attempt.
await settlePendingFinalDelivery(
completion,
"queued",
expectedPendingFinalState ? ["prepared", "queued"] : undefined,
)
: conversationResult(
markConversationDeliveryQueued(
scopeForCompletion(completion),
completion.operationId,
queueId,
),
);
}
/** Finalizes owner state from identified platform evidence before queue acknowledgement. */
export function completeDurableDelivery(
export async function completeDurableDelivery(
completion: DurableDeliveryCompletion,
result: OutboundDeliveryResult,
): ConversationDeliveryRecord {
return markConversationDeliverySent(
scopeForCompletion(completion),
completion.operationId,
readPlatformMessageId(result),
);
stateDir?: string,
): Promise<DurableDeliveryCompletionResult> {
return completion.kind === "pending-final"
? await settlePendingFinalDelivery(completion, "delivered", undefined, stateDir)
: conversationResult(
markConversationDeliverySent(
scopeForCompletion(completion),
completion.operationId,
readPlatformMessageId(result),
),
);
}
/** Finalizes a policy-suppressed send before its durable intent is acknowledged. */
export function suppressDurableDelivery(
export async function suppressDurableDelivery(
completion: DurableDeliveryCompletion,
): ConversationDeliveryRecord {
return markConversationDeliverySuppressed(scopeForCompletion(completion), completion.operationId);
stateDir?: string,
): Promise<DurableDeliveryCompletionResult> {
return completion.kind === "pending-final"
? await settlePendingFinalDelivery(completion, "suppressed", undefined, stateDir)
: conversationResult(
markConversationDeliverySuppressed(scopeForCompletion(completion), completion.operationId),
);
}
/** Finalizes a permanent provider rejection that provably preceded platform I/O. */
export function rejectDurableDelivery(
export async function rejectDurableDelivery(
completion: DurableDeliveryCompletion,
error: string,
): ConversationDeliveryRecord {
return markConversationDeliveryRejected(
scopeForCompletion(completion),
completion.operationId,
error,
);
stateDir?: string,
): Promise<DurableDeliveryCompletionResult> {
return completion.kind === "pending-final"
? await settlePendingFinalDelivery(completion, "unknown", undefined, stateDir)
: conversationResult(
markConversationDeliveryRejected(
scopeForCompletion(completion),
completion.operationId,
error,
),
);
}
/** Makes a dead-lettered durable send terminal without allowing a blind replay. */
export function failDurableDelivery(
export async function failDurableDelivery(
completion: DurableDeliveryCompletion,
): ConversationDeliveryRecord {
return markConversationDeliveryUnknown(scopeForCompletion(completion), completion.operationId);
stateDir?: string,
): Promise<DurableDeliveryCompletionResult> {
return completion.kind === "pending-final"
? await settlePendingFinalDelivery(completion, "unknown", undefined, stateDir)
: conversationResult(
markConversationDeliveryUnknown(scopeForCompletion(completion), completion.operationId),
);
}
@@ -329,6 +329,7 @@ describe("outbound prepared queue migration", () => {
expect(hookMocks.runMessageSending).not.toHaveBeenCalled();
expect(completionMocks.failDurableDelivery).toHaveBeenCalledWith(
interrupted.deliveryCompletion,
tmpDir(),
);
expect(getDeliveryQueueEntryStatus(OUTBOUND_LEGACY_PREPARATION_QUEUE_NAME, id, tmpDir())).toBe(
"failed",
@@ -320,7 +320,7 @@ async function failInterruptedLegacyPreparation(params: {
}
if (params.entry.deliveryCompletion) {
try {
failDurableDelivery(params.entry.deliveryCompletion);
await failDurableDelivery(params.entry.deliveryCompletion, params.stateDir);
} catch (error) {
params.log.warn(
`Legacy delivery ${params.entry.id} interrupted preparation owner could not be marked unknown: ${String(error)}`,
+36 -30
View File
@@ -355,7 +355,7 @@ async function applyRecoveryDeliveryAdmission(params: {
if (admission.status === "allowed") {
return "allowed";
}
markDurableDeliveryFailedBestEffort(params.entry, params.log);
await markDurableDeliveryFailedBestEffort(params.entry, params.log, params.stateDir);
const result = await failPendingDelivery(
{
id: params.entry.id,
@@ -547,7 +547,7 @@ async function moveEntryToFailedWithLogging(
log: RecoveryLogger,
stateDir?: string,
): Promise<boolean> {
markDurableDeliveryFailedBestEffort(entry, log);
await markDurableDeliveryFailedBestEffort(entry, log, stateDir);
try {
const attemptId = recoveryPlatformAttemptId(entry);
await moveEntryToFailedAndCleanup({ entry, cfg, log, stateDir, attemptId });
@@ -599,12 +599,16 @@ async function recordRecoveredFailure(
}).fail(record, error);
}
function markDurableDeliveryFailedBestEffort(entry: QueuedDelivery, log: RecoveryLogger): void {
async function markDurableDeliveryFailedBestEffort(
entry: QueuedDelivery,
log: RecoveryLogger,
stateDir?: string,
): Promise<void> {
if (!entry.deliveryCompletion) {
return;
}
try {
failDurableDelivery(entry.deliveryCompletion);
await failDurableDelivery(entry.deliveryCompletion, stateDir);
} catch (error) {
// Queue ownership is authoritative for replay safety. Missing owner state
// must not leave a dead-lettered delivery permanently replayable.
@@ -626,9 +630,9 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
if (!completion) {
return "continue";
}
let operation: ReturnType<typeof markDurableDeliveryQueued>;
let operation: Awaited<ReturnType<typeof markDurableDeliveryQueued>>;
try {
operation = markDurableDeliveryQueued(completion, opts.entry.id);
operation = await markDurableDeliveryQueued(completion, opts.entry.id);
} catch (error) {
const errMsg = `delivery owner state unavailable: ${formatErrorMessage(error)}`;
await recordRecoveredFailure(failDelivery, opts.entry, errMsg, opts.stateDir).catch(
@@ -638,7 +642,7 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
opts.log.warn(`Delivery entry ${opts.entry.id} ${errMsg}`);
return "failed";
}
if (operation.status === "sent" || operation.status === "replied") {
if (operation.state === "delivered") {
try {
await ackRecoveredDelivery(opts.entry, opts.stateDir);
} catch (error) {
@@ -647,7 +651,7 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
opts.log.warn(`Delivery entry ${opts.entry.id} ${errMsg}`);
return "failed";
}
const messageId = operation.platformMessageId ?? operation.preparedMessageId;
const messageId = operation.platformMessageId;
if (messageId) {
const result: OutboundDeliveryResult = { channel: opts.entry.channel, messageId };
emitRecoveredTerminalSuccess(opts.entry, result);
@@ -663,7 +667,7 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
opts.onRecovered?.(opts.entry);
return "recovered";
}
if (operation.status === "suppressed") {
if (operation.state === "suppressed" || operation.state === "stale") {
try {
await (typeof opts.entry.completionRetention === "object"
? ackRecoveredDelivery(opts.entry, opts.stateDir, { suppressCompletionReceipt: true })
@@ -677,7 +681,7 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
opts.onRecovered?.(opts.entry);
return "recovered";
}
if (operation.status === "rejected") {
if (operation.state === "rejected") {
try {
await (typeof opts.entry.completionRetention === "object"
? ackRecoveredDelivery(opts.entry, opts.stateDir, { suppressCompletionReceipt: true })
@@ -696,17 +700,13 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
failureStage: "platform_send",
}),
);
emitRecoveredTerminalFailure(
opts.entry,
operation.rejectionError ?? "delivery permanently rejected before platform dispatch",
);
opts.onFailed?.(
opts.entry,
operation.rejectionError ?? "delivery permanently rejected before platform dispatch",
);
const error =
operation.rejectionError ?? "delivery permanently rejected before platform dispatch";
emitRecoveredTerminalFailure(opts.entry, error);
opts.onFailed?.(opts.entry, error);
return "failed";
}
if (operation.status === "unknown") {
if (operation.state === "unknown") {
const moved = await moveEntryToFailedWithLogging(opts.entry, opts.cfg, opts.log, opts.stateDir);
return moved ? "moved-to-failed" : "failed";
}
@@ -773,7 +773,7 @@ async function drainQueuedEntry(opts: {
try {
const result = buildReconciledSentResult(entry, reconciliation);
if (entry.deliveryCompletion) {
completeDurableDelivery(entry.deliveryCompletion, result);
await completeDurableDelivery(entry.deliveryCompletion, result, opts.stateDir);
}
await ackRecoveredDelivery(entry, opts.stateDir, undefined, entry.platformSendAttemptId);
emitRecoveredTerminalSuccess(entry, result);
@@ -850,7 +850,7 @@ async function drainQueuedEntry(opts: {
return "failed";
}
try {
markDurableDeliveryFailedBestEffort(entry, opts.log);
await markDurableDeliveryFailedBestEffort(entry, opts.log, opts.stateDir);
const attemptId = recoveryPlatformAttemptId(entry);
await moveEntryToFailedAndCleanup({
entry,
@@ -927,7 +927,7 @@ async function drainQueuedEntry(opts: {
: await reserveDeliveryAttempt(entry.id, maxRetries, opts.stateDir);
if (reservation.status === "exhausted") {
const errMsg = `delivery retry budget exhausted (${reservation.attemptCount}/${maxRetries})`;
markDurableDeliveryFailedBestEffort(entry, opts.log);
await markDurableDeliveryFailedBestEffort(entry, opts.log, opts.stateDir);
try {
await moveEntryToFailedAndCleanup({
entry,
@@ -993,11 +993,6 @@ async function drainQueuedEntry(opts: {
}
if (results.length > 0) {
deliveredResults = [...results];
if (entry.deliveryCompletion) {
completeDurableDelivery(entry.deliveryCompletion, results.at(-1)!);
}
} else if (entry.deliveryCompletion) {
suppressDurableDelivery(entry.deliveryCompletion);
}
const failedOutcomes = payloadOutcomes.filter((outcome) => outcome.status === "failed");
const failedOutcome = failedOutcomes[0];
@@ -1035,6 +1030,13 @@ async function drainQueuedEntry(opts: {
}
return "failed";
}
if (entry.deliveryCompletion) {
if (results.length > 0) {
await completeDurableDelivery(entry.deliveryCompletion, results.at(-1)!, opts.stateDir);
} else {
await suppressDurableDelivery(entry.deliveryCompletion, opts.stateDir);
}
}
postSendState ??=
results.length > 0
? await persistRecoveredPostSendState({
@@ -1155,9 +1157,13 @@ async function drainQueuedEntry(opts: {
if (permanentPlatformRejection || isPermanentDeliveryError(errMsg)) {
try {
if (permanentPlatformRejection && entry.deliveryCompletion) {
rejectDurableDelivery(entry.deliveryCompletion, permanentPlatformRejection.message);
await rejectDurableDelivery(
entry.deliveryCompletion,
permanentPlatformRejection.message,
opts.stateDir,
);
} else {
markDurableDeliveryFailedBestEffort(entry, opts.log);
await markDurableDeliveryFailedBestEffort(entry, opts.log, opts.stateDir);
}
await moveEntryToFailedAndCleanup({
entry,
@@ -1256,7 +1262,7 @@ export async function drainPendingDeliveries(opts: {
!needsUnknownSendReconciliation(currentEntry)
) {
try {
markDurableDeliveryFailedBestEffort(currentEntry, opts.log);
await markDurableDeliveryFailedBestEffort(currentEntry, opts.log, opts.stateDir);
const attemptId = recoveryPlatformAttemptId(currentEntry);
await moveEntryToFailedAndCleanup({
entry: currentEntry,