mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-13 22:23:08 -06:00
fix(gateway): prevent restart replay after final delivery (#121908)
Re-lands the reverted #121507 on the current custody contracts. The reply dispatcher now owns direct-send pending-final custody: claim before provider I/O, terminal settlement for delivered/suppressed/failed outcomes, proven no-send stays replayable, ambiguous evidence fails closed — so Gateway restarts can no longer duplicate an already-accepted final reply. Proof: ClawSweeper local review clean, exact-head ci-gate green, live Telegram E2E (one turn, one final, no duplicates). Co-authored-by: Ayaan Zaidi <hi@obviy.us>
This commit is contained in:
+237
-237
File diff suppressed because it is too large
Load Diff
@@ -323,6 +323,7 @@ export async function dispatchDiscordComponentEvent(params: {
|
||||
chunkMode: resolveChunkMode(ctx.cfg, "discord", accountId),
|
||||
mediaLocalRoots,
|
||||
kind: info.kind,
|
||||
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
|
||||
});
|
||||
if (result.visibleReplySent) {
|
||||
replyReference.markSent();
|
||||
|
||||
@@ -271,7 +271,10 @@ async function processDiscordMessageInner(
|
||||
|
||||
const deliverDiscordPayload = async (
|
||||
payload: ReplyPayload,
|
||||
info: { kind: ReplyDispatchKind },
|
||||
info: {
|
||||
kind: ReplyDispatchKind;
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
|
||||
},
|
||||
options?: {
|
||||
allowFallbackOnlyToolWarning?: boolean;
|
||||
allowProgressBlock?: boolean;
|
||||
@@ -328,6 +331,7 @@ async function processDiscordMessageInner(
|
||||
threadBindings,
|
||||
mediaLocalRoots,
|
||||
kind: "block",
|
||||
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
|
||||
});
|
||||
if (result.visibleReplySent) {
|
||||
replyReference.markSent();
|
||||
@@ -484,6 +488,7 @@ async function processDiscordMessageInner(
|
||||
mediaLocalRoots,
|
||||
allowedMentions,
|
||||
kind: info.kind,
|
||||
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
|
||||
});
|
||||
return deliveryResult.visibleReplySent;
|
||||
},
|
||||
@@ -542,6 +547,7 @@ async function processDiscordMessageInner(
|
||||
threadBindings,
|
||||
mediaLocalRoots,
|
||||
kind: info.kind,
|
||||
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
|
||||
});
|
||||
if (!result.visibleReplySent) {
|
||||
return result;
|
||||
|
||||
@@ -229,13 +229,16 @@ export async function deliverDiscordReply(params: {
|
||||
mediaLocalRoots?: readonly string[];
|
||||
allowedMentions?: DiscordAllowedMentions;
|
||||
kind: "tool" | "block" | "final";
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
|
||||
}) {
|
||||
void params.runtime;
|
||||
|
||||
const delivery = resolveDiscordDeliveryOptions(params);
|
||||
const payloads = sanitizeDiscordFrontChannelReplyPayloads(params.replies, {
|
||||
kind: params.kind,
|
||||
}).map(formatDiscordReasoningPayload);
|
||||
})
|
||||
.map(formatDiscordReasoningPayload)
|
||||
.map((payload) => params.bindPendingFinalDelivery?.(payload) ?? payload);
|
||||
if (payloads.length === 0) {
|
||||
return {
|
||||
visibleReplySent: false,
|
||||
|
||||
@@ -86,9 +86,7 @@ export function sanitizeDiscordFrontChannelReplyPayloads(
|
||||
: sanitizeDiscordFrontChannelText(payload.text)
|
||||
: payload.text;
|
||||
const nextPayload =
|
||||
safeText === payload.text
|
||||
? payload
|
||||
: ({ ...payload, text: safeText || undefined } as ReplyPayload);
|
||||
safeText === payload.text ? payload : { ...payload, text: safeText || undefined };
|
||||
const nextParts = resolveSendableOutboundReplyParts(nextPayload);
|
||||
if (!nextParts.hasContent && !hasNonTextReplyPayloadContent(nextPayload)) {
|
||||
continue;
|
||||
|
||||
@@ -65,6 +65,7 @@ async function maybeSendDiscordWebhookText(params: {
|
||||
accountId?: string | null;
|
||||
identity?: OutboundIdentity;
|
||||
replyToId?: string | null;
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
}): Promise<{ messageId: string; channelId: string } | null> {
|
||||
if (params.threadId == null) {
|
||||
return null;
|
||||
@@ -96,6 +97,7 @@ async function maybeSendDiscordWebhookText(params: {
|
||||
replyTo: params.replyToId ?? undefined,
|
||||
username: persona.username,
|
||||
avatarUrl: persona.avatarUrl,
|
||||
onPlatformSendDispatch: params.onPlatformSendDispatch,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
@@ -130,6 +132,7 @@ async function resolveDiscordOutboundMessageSend(params: DiscordOutboundMessageC
|
||||
await params.onDeliveryResult?.(attachChannelToResult("discord", result));
|
||||
}
|
||||
: undefined,
|
||||
onPlatformSendDispatch: params.onPlatformSendDispatch,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -173,16 +176,29 @@ export const discordOutbound: ChannelOutboundAdapter = {
|
||||
channel: "discord",
|
||||
sendText: async (ctx) => {
|
||||
if (!ctx.silent) {
|
||||
const webhookResult = await maybeSendDiscordWebhookText({
|
||||
cfg: ctx.cfg,
|
||||
text: ctx.text,
|
||||
threadId: ctx.threadId,
|
||||
accountId: ctx.accountId,
|
||||
identity: ctx.identity,
|
||||
replyToId: ctx.replyToId,
|
||||
}).catch(() => null);
|
||||
if (webhookResult) {
|
||||
return webhookResult;
|
||||
let webhookSelected = false;
|
||||
try {
|
||||
const webhookResult = await maybeSendDiscordWebhookText({
|
||||
cfg: ctx.cfg,
|
||||
text: ctx.text,
|
||||
threadId: ctx.threadId,
|
||||
accountId: ctx.accountId,
|
||||
identity: ctx.identity,
|
||||
replyToId: ctx.replyToId,
|
||||
onPlatformSendDispatch: ctx.onPlatformSendDispatch
|
||||
? async () => {
|
||||
webhookSelected = true;
|
||||
await ctx.onPlatformSendDispatch?.();
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
if (webhookResult) {
|
||||
return webhookResult;
|
||||
}
|
||||
} catch (error) {
|
||||
if (webhookSelected) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
const { send, target, options } = await resolveDiscordOutboundMessageSend(ctx);
|
||||
@@ -202,6 +218,7 @@ export const discordOutbound: ChannelOutboundAdapter = {
|
||||
mediaAccess: ctx.mediaAccess,
|
||||
mediaLocalRoots: ctx.mediaLocalRoots,
|
||||
mediaReadFile: ctx.mediaReadFile,
|
||||
onPlatformSendDispatch: ctx.onPlatformSendDispatch,
|
||||
});
|
||||
}
|
||||
const mediaOptions = {
|
||||
@@ -235,13 +252,14 @@ export const discordOutbound: ChannelOutboundAdapter = {
|
||||
}
|
||||
return await send(target, ctx.text, mediaOptions);
|
||||
},
|
||||
sendPoll: async ({ cfg, to, poll, accountId, threadId, silent }) =>
|
||||
sendPoll: async ({ cfg, to, poll, accountId, threadId, silent, onPlatformSendDispatch }) =>
|
||||
await (
|
||||
await loadDiscordSendRuntime()
|
||||
).sendPollDiscord(resolveDiscordOutboundTarget({ to, threadId }), poll, {
|
||||
accountId: accountId ?? undefined,
|
||||
silent: silent ?? undefined,
|
||||
cfg,
|
||||
onPlatformSendDispatch,
|
||||
}),
|
||||
}),
|
||||
afterDeliverPayload: async ({ cfg, target, payload, results }) => {
|
||||
|
||||
@@ -57,6 +57,7 @@ function resolveDiscordDeliveryOptions(
|
||||
accountId: ctx.accountId ?? undefined,
|
||||
silent: ctx.silent ?? undefined,
|
||||
cfg: ctx.cfg,
|
||||
onPlatformSendDispatch: ctx.onPlatformSendDispatch,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -172,6 +172,7 @@ type DiscordComponentSendOpts = {
|
||||
allowedMentions?: DiscordAllowedMentions;
|
||||
/** Persist the concrete platform send before component bookkeeping can fail. */
|
||||
onDeliveryResult?: (result: DiscordSendResult) => Promise<void> | void;
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
};
|
||||
|
||||
export function registerBuiltDiscordComponentMessage(params: {
|
||||
@@ -291,6 +292,7 @@ export async function sendDiscordComponentMessage(
|
||||
tableMode: opts.tableMode,
|
||||
chunkMode: opts.chunkMode,
|
||||
onDeliveryResult: opts.onDeliveryResult,
|
||||
onPlatformSendDispatch: opts.onPlatformSendDispatch,
|
||||
...(opts.suppressEmbeds === undefined ? {} : { suppressEmbeds: opts.suppressEmbeds }),
|
||||
});
|
||||
}
|
||||
@@ -321,6 +323,7 @@ export async function sendDiscordComponentMessage(
|
||||
|
||||
let result: { id: string; channel_id: string };
|
||||
try {
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
result = (await request(
|
||||
() =>
|
||||
createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, {
|
||||
|
||||
@@ -70,6 +70,8 @@ type DiscordSendOpts = {
|
||||
allowedMentions?: DiscordAllowedMentions;
|
||||
/** Persist each concrete platform send before any later chunk can fail. */
|
||||
onDeliveryResult?: (result: DiscordSendResult) => Promise<void> | void;
|
||||
/** @internal Refresh durable custody immediately before Discord REST I/O. */
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
};
|
||||
|
||||
type DiscordClientRequest = ReturnType<typeof createDiscordClient>["request"];
|
||||
@@ -92,6 +94,7 @@ async function sendDiscordThreadTextChunks(params: {
|
||||
suppressEmbeds?: boolean;
|
||||
allowedMentions?: DiscordAllowedMentions;
|
||||
onResult?: DiscordSendProgress;
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
}): Promise<void> {
|
||||
for (const chunk of params.chunks) {
|
||||
await sendDiscordText({
|
||||
@@ -106,6 +109,7 @@ async function sendDiscordThreadTextChunks(params: {
|
||||
allowedMentions: params.allowedMentions,
|
||||
maxChars: params.maxChars,
|
||||
onResult: params.onResult,
|
||||
onPlatformSendDispatch: params.onPlatformSendDispatch,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -238,6 +242,7 @@ export async function sendMessageDiscord(
|
||||
});
|
||||
let threadRes: { id: string; message?: { id: string; channel_id: string } };
|
||||
try {
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
threadRes = (await request(
|
||||
() =>
|
||||
createThread<{ id: string; message?: { id: string; channel_id: string } }>(
|
||||
@@ -309,6 +314,7 @@ export async function sendMessageDiscord(
|
||||
allowedMentions: opts.allowedMentions,
|
||||
maxChars: textLimit,
|
||||
onResult: reportThreadResult,
|
||||
onPlatformSendDispatch: opts.onPlatformSendDispatch,
|
||||
});
|
||||
await sendDiscordThreadTextChunks({
|
||||
rest,
|
||||
@@ -322,6 +328,7 @@ export async function sendMessageDiscord(
|
||||
suppressEmbeds,
|
||||
allowedMentions: opts.allowedMentions,
|
||||
onResult: reportThreadResult,
|
||||
onPlatformSendDispatch: opts.onPlatformSendDispatch,
|
||||
});
|
||||
} else {
|
||||
await sendDiscordThreadTextChunks({
|
||||
@@ -336,6 +343,7 @@ export async function sendMessageDiscord(
|
||||
suppressEmbeds,
|
||||
allowedMentions: opts.allowedMentions,
|
||||
onResult: reportThreadResult,
|
||||
onPlatformSendDispatch: opts.onPlatformSendDispatch,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -391,6 +399,7 @@ export async function sendMessageDiscord(
|
||||
allowedMentions: opts.allowedMentions,
|
||||
maxChars: textLimit,
|
||||
onResult: reportResult,
|
||||
onPlatformSendDispatch: opts.onPlatformSendDispatch,
|
||||
});
|
||||
} else {
|
||||
result = await sendDiscordText({
|
||||
@@ -408,6 +417,7 @@ export async function sendMessageDiscord(
|
||||
allowedMentions: opts.allowedMentions,
|
||||
maxChars: textLimit,
|
||||
onResult: reportResult,
|
||||
onPlatformSendDispatch: opts.onPlatformSendDispatch,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -447,6 +457,7 @@ export async function sendStickerDiscord(
|
||||
enforce_nonce: true,
|
||||
...(flags ? { flags } : {}),
|
||||
};
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
const res = (await request(
|
||||
() => createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, { body }),
|
||||
"sticker",
|
||||
@@ -474,6 +485,7 @@ export async function sendPollDiscord(
|
||||
enforce_nonce: true,
|
||||
...(flags ? { flags } : {}),
|
||||
};
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
const res = (await request(
|
||||
() => createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, { body }),
|
||||
"poll",
|
||||
|
||||
@@ -325,6 +325,7 @@ type DiscordTextSendParams = {
|
||||
suppressEmbeds?: boolean;
|
||||
maxChars?: number;
|
||||
onResult?: DiscordSendProgress;
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
};
|
||||
|
||||
async function sendDiscordText(params: DiscordTextSendParams) {
|
||||
@@ -343,6 +344,7 @@ async function sendDiscordText(params: DiscordTextSendParams) {
|
||||
suppressEmbeds,
|
||||
maxChars,
|
||||
onResult,
|
||||
onPlatformSendDispatch,
|
||||
} = params;
|
||||
if (!text.trim()) {
|
||||
throw new Error("Message must be non-empty for Discord sends");
|
||||
@@ -369,6 +371,7 @@ async function sendDiscordText(params: DiscordTextSendParams) {
|
||||
flags,
|
||||
replyTo: chunkReplyTo,
|
||||
});
|
||||
await onPlatformSendDispatch?.();
|
||||
const result = (await request(
|
||||
() => createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, { body }),
|
||||
"text",
|
||||
@@ -429,6 +432,7 @@ async function sendDiscordMedia(params: DiscordMediaSendParams) {
|
||||
suppressEmbeds,
|
||||
maxChars,
|
||||
onResult,
|
||||
onPlatformSendDispatch,
|
||||
} = params;
|
||||
const media = await loadWebMedia(
|
||||
mediaUrl,
|
||||
@@ -472,6 +476,7 @@ async function sendDiscordMedia(params: DiscordMediaSendParams) {
|
||||
});
|
||||
let res: { id: string; channel_id: string };
|
||||
try {
|
||||
await onPlatformSendDispatch?.();
|
||||
res = (await request(
|
||||
() => createChannelMessage<{ id: string; channel_id: string }>(rest, channelId, { body }),
|
||||
"media",
|
||||
@@ -496,6 +501,7 @@ async function sendDiscordMedia(params: DiscordMediaSendParams) {
|
||||
allowedMentions,
|
||||
maxChars,
|
||||
onResult,
|
||||
onPlatformSendDispatch,
|
||||
});
|
||||
}
|
||||
await onResult?.(res, "media", reply?.messageId);
|
||||
|
||||
@@ -38,6 +38,7 @@ type VoiceMessageOpts = Pick<
|
||||
| "mediaAccess"
|
||||
| "mediaLocalRoots"
|
||||
| "mediaReadFile"
|
||||
| "onPlatformSendDispatch"
|
||||
>;
|
||||
|
||||
function toDiscordSendResult(
|
||||
@@ -122,6 +123,7 @@ export async function sendVoiceMessageDiscord(
|
||||
|
||||
const metadata = await getVoiceMessageMetadata(oggPath);
|
||||
const audioBuffer = await fs.readFile(oggPath);
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
const result = await sendDiscordVoiceMessage(
|
||||
rest,
|
||||
channelId,
|
||||
|
||||
@@ -40,6 +40,7 @@ type DiscordWebhookSendOpts = {
|
||||
username?: string;
|
||||
avatarUrl?: string;
|
||||
wait?: boolean;
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
};
|
||||
|
||||
function resolveWebhookExecutionUrl(params: {
|
||||
@@ -154,6 +155,7 @@ export async function sendWebhookMessageDiscord(
|
||||
try {
|
||||
const response = await request(
|
||||
async () => {
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
const attemptResponse = await (proxyFetch ?? fetch)(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
|
||||
@@ -208,7 +208,10 @@ export function createTelegramDeliveryController(params: {
|
||||
) {
|
||||
return payload;
|
||||
}
|
||||
return { ...payload, replyToId: implicitQuoteReplyTargetId };
|
||||
return {
|
||||
...payload,
|
||||
replyToId: implicitQuoteReplyTargetId,
|
||||
};
|
||||
};
|
||||
const usesNativeTelegramQuote = (payload: ReplyPayload): boolean =>
|
||||
params.replyQuoteText != null ||
|
||||
@@ -223,6 +226,8 @@ export function createTelegramDeliveryController(params: {
|
||||
mirrorTranscript?: boolean;
|
||||
promptContextSequence?: TelegramPromptContextProjectionSequence;
|
||||
textMode?: "html";
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
|
||||
},
|
||||
) => {
|
||||
if (params.isDispatchSuperseded()) {
|
||||
@@ -253,10 +258,13 @@ export function createTelegramDeliveryController(params: {
|
||||
)
|
||||
: undefined,
|
||||
);
|
||||
const effectivePayload = withTelegramPromptContextSource(
|
||||
const projectedPayload = withTelegramPromptContextSource(
|
||||
deliverablePayload,
|
||||
projectionSequence.source,
|
||||
);
|
||||
const effectivePayload = options?.bindPendingFinalDelivery
|
||||
? options.bindPendingFinalDelivery(projectedPayload)
|
||||
: projectedPayload;
|
||||
const silent =
|
||||
options?.silent ??
|
||||
(params.telegramCfg.silentErrorReplies === true && payload.isError === true);
|
||||
@@ -316,6 +324,7 @@ export function createTelegramDeliveryController(params: {
|
||||
silent,
|
||||
mediaLoader: params.telegramDeps.loadWebMedia,
|
||||
promptContextSequence: projectionSequence,
|
||||
onPlatformSendDispatch: options?.onPlatformSendDispatch,
|
||||
...(options?.textMode ? { textMode: options.textMode } : {}),
|
||||
});
|
||||
if (!result.delivered) {
|
||||
@@ -432,6 +441,8 @@ export function createTelegramDeliveryController(params: {
|
||||
payload: ReplyPayload,
|
||||
text: string,
|
||||
promptContextSequence: TelegramPromptContextProjectionSequence,
|
||||
onPlatformSendDispatch?: () => Promise<void>,
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T,
|
||||
): Promise<LaneDeliveryResult> => {
|
||||
const afterAcceptedDraft = params.draft.answerLane.stream?.hasConsumedReplyTarget?.() === true;
|
||||
if (payload.isError === true) {
|
||||
@@ -441,6 +452,8 @@ export function createTelegramDeliveryController(params: {
|
||||
afterAcceptedDraft,
|
||||
durable: true,
|
||||
promptContextSequence,
|
||||
onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery,
|
||||
});
|
||||
if (!delivered) {
|
||||
return { kind: "skipped" };
|
||||
@@ -454,6 +467,8 @@ export function createTelegramDeliveryController(params: {
|
||||
afterAcceptedDraft,
|
||||
durable: true,
|
||||
promptContextSequence,
|
||||
onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery,
|
||||
});
|
||||
if (barLine) {
|
||||
await params.progress.applyCollapseSummary(barLine, postCosmeticSummaryBar);
|
||||
@@ -472,6 +487,8 @@ export function createTelegramDeliveryController(params: {
|
||||
answerPayload: ReplyPayload,
|
||||
text: string,
|
||||
buttons?: TelegramInlineButtons,
|
||||
onPlatformSendDispatch?: () => Promise<void>,
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T,
|
||||
): Promise<LaneDeliveryResult> => {
|
||||
const transcriptFinal = await resolveCurrentTurnTranscriptFinal();
|
||||
const finalText = await resolveTranscriptBackedChannelFinalText({
|
||||
@@ -491,6 +508,8 @@ export function createTelegramDeliveryController(params: {
|
||||
answerPayload,
|
||||
finalText,
|
||||
promptContextSequence,
|
||||
onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery,
|
||||
);
|
||||
} else {
|
||||
if (isFollowUp) {
|
||||
@@ -506,6 +525,8 @@ export function createTelegramDeliveryController(params: {
|
||||
buttons,
|
||||
allowStream: !usesNativeTelegramQuote(answerPayload),
|
||||
promptContextSequence,
|
||||
onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery,
|
||||
});
|
||||
if (!isFollowUp && result.kind !== "skipped") {
|
||||
params.progress.markFinalDelivered();
|
||||
|
||||
@@ -9,8 +9,8 @@ import {
|
||||
isFastModeAutoProgressPayload,
|
||||
isReplyPayloadNonTerminalToolErrorWarning,
|
||||
resolveSendableOutboundReplyParts,
|
||||
type ReplyPayload,
|
||||
} from "openclaw/plugin-sdk/reply-payload";
|
||||
import type { ReplyPayload } from "openclaw/plugin-sdk/reply-payload";
|
||||
import type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { danger } from "openclaw/plugin-sdk/runtime-env";
|
||||
import type { TelegramBotDeps } from "./bot-deps.js";
|
||||
@@ -105,6 +105,8 @@ export function createTelegramReplyDelivery(params: {
|
||||
| {
|
||||
promise: Promise<{ visibleReplySent: boolean }>;
|
||||
visibleReplySent: boolean;
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
|
||||
resolve: (result: { visibleReplySent: boolean }) => void;
|
||||
reject: (error: unknown) => void;
|
||||
}
|
||||
@@ -144,6 +146,8 @@ export function createTelegramReplyDelivery(params: {
|
||||
buffered.payload,
|
||||
buffered.text,
|
||||
resolvePayloadTelegramInlineButtons(buffered.payload),
|
||||
settlement?.onPlatformSendDispatch,
|
||||
settlement?.bindPendingFinalDelivery,
|
||||
);
|
||||
if (settlement) {
|
||||
settlement.resolve({
|
||||
@@ -271,6 +275,8 @@ export function createTelegramReplyDelivery(params: {
|
||||
bufferedFinalSettlement = {
|
||||
promise: finalization,
|
||||
visibleReplySent: blockDelivered,
|
||||
onPlatformSendDispatch: info.onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
|
||||
resolve: resolveFinalization,
|
||||
reject: rejectFinalization,
|
||||
};
|
||||
@@ -381,6 +387,8 @@ export function createTelegramReplyDelivery(params: {
|
||||
effectivePayload,
|
||||
segment.update.text,
|
||||
telegramButtons,
|
||||
info.onPlatformSendDispatch,
|
||||
info.bindPendingFinalDelivery,
|
||||
)
|
||||
: await params.delivery.deliverLaneText({
|
||||
laneName: segment.lane,
|
||||
@@ -389,6 +397,8 @@ export function createTelegramReplyDelivery(params: {
|
||||
infoKind: info.kind,
|
||||
buttons: telegramButtons,
|
||||
allowStream: !isDurableProgressCommentary,
|
||||
onPlatformSendDispatch: info.onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
|
||||
});
|
||||
if (
|
||||
segment.lane === "answer" &&
|
||||
@@ -440,6 +450,8 @@ export function createTelegramReplyDelivery(params: {
|
||||
: effectivePayload;
|
||||
delivered = await params.delivery.sendPayload(payloadWithoutReasoning, {
|
||||
durable: info.kind === "final",
|
||||
onPlatformSendDispatch: info.onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
|
||||
});
|
||||
}
|
||||
if (info.kind === "final" && delivered) {
|
||||
@@ -463,6 +475,8 @@ export function createTelegramReplyDelivery(params: {
|
||||
}
|
||||
const delivered = await params.delivery.sendPayload(effectivePayload, {
|
||||
durable: info.kind === "final",
|
||||
onPlatformSendDispatch: info.onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery: info.bindPendingFinalDelivery,
|
||||
});
|
||||
if (info.kind === "final" && delivered) {
|
||||
params.progress.markFinalDelivered();
|
||||
|
||||
@@ -1777,7 +1777,7 @@ export const registerTelegramNativeCommands = ({
|
||||
},
|
||||
},
|
||||
delivery: {
|
||||
deliverWithProviderMessageSending: async (payload) => {
|
||||
deliverWithProviderMessageSending: async (payload, info) => {
|
||||
if (
|
||||
shouldSuppressLocalTelegramExecApprovalPrompt({
|
||||
cfg: runtimeCfg,
|
||||
@@ -1802,6 +1802,7 @@ export const registerTelegramNativeCommands = ({
|
||||
],
|
||||
...deliveryBaseOptions,
|
||||
silent: runtimeTelegramCfg.silentErrorReplies === true && payload.isError === true,
|
||||
onPlatformSendDispatch: info.onPlatformSendDispatch,
|
||||
});
|
||||
if (result.delivered) {
|
||||
deliveryState.delivered = true;
|
||||
|
||||
@@ -71,16 +71,21 @@ export function createTelegramNativeCommandTestDeps(
|
||||
): { dispatchChannelInboundTurn: DispatchChannelInboundTurn } {
|
||||
return {
|
||||
dispatchChannelInboundTurn: async (plan) => {
|
||||
const delivery = plan.delivery;
|
||||
const dispatchResult = await dispatchReply({
|
||||
ctx: plan.ctxPayload,
|
||||
cfg: plan.cfg,
|
||||
dispatcherOptions: {
|
||||
...plan.dispatcherOptions,
|
||||
deliver:
|
||||
"deliverWithProviderMessageSending" in plan.delivery
|
||||
? plan.delivery.deliverWithProviderMessageSending
|
||||
: plan.delivery.deliver,
|
||||
onError: plan.delivery.onError,
|
||||
"deliverWithProviderMessageSending" in delivery
|
||||
? (payload, info) =>
|
||||
delivery.deliverWithProviderMessageSending(payload, {
|
||||
...info,
|
||||
onPlatformSendDispatch: info.onPlatformSendDispatch ?? (async () => undefined),
|
||||
})
|
||||
: delivery.deliver,
|
||||
onError: delivery.onError,
|
||||
},
|
||||
replyOptions: plan.replyOptions,
|
||||
});
|
||||
|
||||
@@ -38,7 +38,11 @@ export async function runTelegramChannelInboundEventWithHarness(
|
||||
cfg: plan.cfg,
|
||||
dispatcherOptions: {
|
||||
...plan.dispatcherOptions,
|
||||
deliver: plan.delivery.deliverWithProviderMessageSending,
|
||||
deliver: (payload, info) =>
|
||||
plan.delivery.deliverWithProviderMessageSending(payload, {
|
||||
...info,
|
||||
onPlatformSendDispatch: info.onPlatformSendDispatch ?? (async () => undefined),
|
||||
}),
|
||||
onError: plan.delivery.onError,
|
||||
},
|
||||
toolsAllow: plan.toolsAllow,
|
||||
|
||||
@@ -254,6 +254,7 @@ async function deliverTextReply(params: {
|
||||
progress: DeliveryProgress;
|
||||
recordMessageId: (messageId: number) => void;
|
||||
quoteOnlyOnFirstChunk?: boolean;
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
}): Promise<number | undefined> {
|
||||
let firstDeliveredMessageId: number | undefined;
|
||||
const chunks = filterEmptyTelegramTextChunks(params.chunkText(params.text));
|
||||
@@ -278,6 +279,7 @@ async function deliverTextReply(params: {
|
||||
markDelivered,
|
||||
sendChunk: async ({ chunk, isFirstChunk, replyToMessageId, replyMarkup, replyQuoteText }) => {
|
||||
const includeQuoteMetadata = params.quoteOnlyOnFirstChunk !== true || isFirstChunk;
|
||||
await params.onPlatformSendDispatch?.();
|
||||
const messageId = await sendTelegramText(
|
||||
params.bot,
|
||||
params.chatId,
|
||||
@@ -351,6 +353,7 @@ async function deliverMediaReply(params: {
|
||||
progress: DeliveryProgress;
|
||||
recordMessageId: (messageId: number) => void;
|
||||
textMode?: "html";
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
}): Promise<{ firstDeliveredMessageId?: number; visibleFallbackText?: string }> {
|
||||
let firstDeliveredMessageId: number | undefined;
|
||||
let visibleFallbackText: string | undefined;
|
||||
@@ -372,6 +375,7 @@ async function deliverMediaReply(params: {
|
||||
plainCaption?: string;
|
||||
shouldLog?: (err: unknown) => boolean;
|
||||
}) => {
|
||||
await params.onPlatformSendDispatch?.();
|
||||
const delivery = await sendTelegramCaptionedMediaWithFallback({
|
||||
operation: options.sender.operation,
|
||||
requestParams: options.requestParams,
|
||||
@@ -517,6 +521,7 @@ async function deliverMediaReply(params: {
|
||||
progress: createVoiceFallbackProgress(),
|
||||
recordMessageId: params.recordMessageId,
|
||||
quoteOnlyOnFirstChunk: true,
|
||||
onPlatformSendDispatch: params.onPlatformSendDispatch,
|
||||
});
|
||||
|
||||
await params.onVoiceRecording?.();
|
||||
@@ -612,6 +617,7 @@ async function deliverMediaReply(params: {
|
||||
replyToMode: params.replyToMode,
|
||||
progress: params.progress,
|
||||
recordMessageId: params.recordMessageId,
|
||||
onPlatformSendDispatch: params.onPlatformSendDispatch,
|
||||
});
|
||||
if (followUpMessageId === undefined) {
|
||||
visibleFallbackText = firstDeliveredCaption ?? "";
|
||||
@@ -783,6 +789,8 @@ export async function deliverReplies(params: {
|
||||
promptContextSequence?: TelegramPromptContextProjectionSequence;
|
||||
/** Text is already prepared Telegram HTML and must not be parsed as Markdown again. */
|
||||
textMode?: "html";
|
||||
/** @internal Claim delivery custody immediately before Telegram Bot API I/O. */
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
}): Promise<{
|
||||
delivered: boolean;
|
||||
}> {
|
||||
@@ -930,6 +938,7 @@ export async function deliverReplies(params: {
|
||||
);
|
||||
let firstDeliveredMessageId: number | undefined;
|
||||
if (reactionEmoji && typeof replyToId === "number") {
|
||||
await params.onPlatformSendDispatch?.();
|
||||
const reactionResult = await reactMessageTelegram(params.chatId, replyToId, reactionEmoji, {
|
||||
cfg: params.cfg ?? { channels: { telegram: { botToken: params.token } } },
|
||||
token: params.token,
|
||||
@@ -966,6 +975,7 @@ export async function deliverReplies(params: {
|
||||
replyToMode: params.replyToMode,
|
||||
progress,
|
||||
recordMessageId,
|
||||
onPlatformSendDispatch: params.onPlatformSendDispatch,
|
||||
});
|
||||
} else if (mediaList.length > 0) {
|
||||
const mediaDelivery = await deliverMediaReply({
|
||||
@@ -993,6 +1003,7 @@ export async function deliverReplies(params: {
|
||||
replyToMode: params.replyToMode,
|
||||
progress,
|
||||
recordMessageId,
|
||||
onPlatformSendDispatch: params.onPlatformSendDispatch,
|
||||
...(params.textMode ? { textMode: params.textMode } : {}),
|
||||
});
|
||||
firstDeliveredMessageId = mediaDelivery.firstDeliveredMessageId;
|
||||
|
||||
@@ -61,7 +61,7 @@ const MAX_PREVIEW_FLOOD_SUSPEND_MS = 60_000;
|
||||
const MIN_PREVIEW_DWELL_MS = 4_000;
|
||||
|
||||
export type TelegramDraftStream = {
|
||||
update: (text: string) => void;
|
||||
update: (text: string, options?: { onPlatformSendDispatch?: () => Promise<void> }) => void;
|
||||
updateLazy: (resolveText: () => string | undefined) => void;
|
||||
updatePreview: (preview: TelegramDraftPreview) => void;
|
||||
flush: () => Promise<void>;
|
||||
@@ -323,6 +323,7 @@ export function createTelegramDraftStream(params: {
|
||||
let lastDeliveredText = "";
|
||||
let lastRequestedText = "";
|
||||
let lastRequestedPreview: TelegramDraftPreview | undefined;
|
||||
let pendingPlatformSendDispatch: (() => Promise<void>) | undefined;
|
||||
let generation = 0;
|
||||
let finalPagePlan: { pages: PlannedTelegramDraftPage[]; nextPageIndex: number } | undefined;
|
||||
// Generations whose in-flight FIRST send was superseded by a reposition
|
||||
@@ -450,6 +451,10 @@ export function createTelegramDraftStream(params: {
|
||||
page: PlannedTelegramDraftPage,
|
||||
sendGeneration: number,
|
||||
): Promise<boolean> => {
|
||||
if (pendingPlatformSendDispatch) {
|
||||
await pendingPlatformSendDispatch();
|
||||
pendingPlatformSendDispatch = undefined;
|
||||
}
|
||||
const targetMessageId = streamMessageId;
|
||||
if (typeof targetMessageId === "number") {
|
||||
streamVisibleSinceMs ??= Date.now();
|
||||
@@ -771,12 +776,17 @@ export function createTelegramDraftStream(params: {
|
||||
throwTerminalDeliveryError();
|
||||
};
|
||||
|
||||
const requestDraftUpdate = (text: string, preview?: TelegramDraftPreview) => {
|
||||
const requestDraftUpdate = (
|
||||
text: string,
|
||||
preview?: TelegramDraftPreview,
|
||||
onPlatformSendDispatch?: () => Promise<void>,
|
||||
) => {
|
||||
if (streamState.stopped || streamState.final) {
|
||||
return;
|
||||
}
|
||||
lastRequestedPreview = preview;
|
||||
lastRequestedText = text;
|
||||
pendingPlatformSendDispatch = onPlatformSendDispatch;
|
||||
updateDraft(text);
|
||||
};
|
||||
|
||||
@@ -849,6 +859,7 @@ export function createTelegramDraftStream(params: {
|
||||
streamState.final = true;
|
||||
observeCurrentProviderMessage();
|
||||
await drainProviderMessageObservations();
|
||||
pendingPlatformSendDispatch = undefined;
|
||||
};
|
||||
|
||||
const remainingFinalContent = (): TelegramDraftMessageSnapshot | undefined => {
|
||||
@@ -1049,7 +1060,7 @@ export function createTelegramDraftStream(params: {
|
||||
params.log?.(`telegram stream preview ready (maxChars=${maxChars}, throttleMs=${throttleMs})`);
|
||||
|
||||
return {
|
||||
update: requestDraftUpdate,
|
||||
update: (text, options) => requestDraftUpdate(text, undefined, options?.onPlatformSendDispatch),
|
||||
updateLazy: requestLazyDraftUpdate,
|
||||
updatePreview,
|
||||
flush,
|
||||
|
||||
@@ -11,8 +11,8 @@ import {
|
||||
buildTtsSupplementMediaPayload,
|
||||
getReplyPayloadTtsSupplement,
|
||||
resolveSendableOutboundReplyParts,
|
||||
type ReplyPayload,
|
||||
} from "openclaw/plugin-sdk/reply-payload";
|
||||
import type { ReplyPayload } from "openclaw/plugin-sdk/reply-runtime";
|
||||
import type { TelegramInlineButtons } from "./button-types.js";
|
||||
import type { TelegramDraftStream } from "./draft-stream.js";
|
||||
import type { TelegramPromptContextProjectionSequence } from "./prompt-context-projection.js";
|
||||
@@ -55,6 +55,8 @@ type CreateLaneTextDelivererParams = {
|
||||
durable?: boolean;
|
||||
promptContextSequence?: TelegramPromptContextProjectionSequence;
|
||||
textMode?: "html";
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
|
||||
},
|
||||
) => Promise<boolean>;
|
||||
flushDraftLane: (lane: DraftLaneState) => Promise<void>;
|
||||
@@ -86,6 +88,8 @@ type DeliverLaneTextParams = {
|
||||
durable?: boolean;
|
||||
allowStream?: boolean;
|
||||
promptContextSequence?: TelegramPromptContextProjectionSequence;
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
|
||||
};
|
||||
|
||||
function result(
|
||||
@@ -276,6 +280,7 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
|
||||
promptContextSequence: TelegramPromptContextProjectionSequence,
|
||||
followedByDurablePayload = false,
|
||||
allowErrorPayload = false,
|
||||
onPlatformSendDispatch?: () => Promise<void>,
|
||||
): Promise<LaneDeliveryResult | undefined> => {
|
||||
const stream = lane.stream;
|
||||
if (!stream || text.length === 0 || (payload.isError && !allowErrorPayload)) {
|
||||
@@ -301,8 +306,15 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
|
||||
lane.lastPartialText = previewText;
|
||||
lane.hasStreamedMessage = true;
|
||||
lane.finalized = false;
|
||||
if (stream.lastDeliveredText?.() !== previewText) {
|
||||
stream.update(previewText);
|
||||
const previewAlreadyVisible = stream.lastDeliveredText?.() === previewText;
|
||||
if (!previewAlreadyVisible) {
|
||||
if (finalizePreview && onPlatformSendDispatch) {
|
||||
stream.update(previewText, { onPlatformSendDispatch });
|
||||
} else {
|
||||
stream.update(previewText);
|
||||
}
|
||||
} else if (finalizePreview) {
|
||||
await onPlatformSendDispatch?.();
|
||||
}
|
||||
if (finalizePreview) {
|
||||
await params.stopDraftLane(lane);
|
||||
@@ -341,6 +353,7 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
|
||||
let buttonsAttached = false;
|
||||
if (buttons && activeSnapshot) {
|
||||
try {
|
||||
await onPlatformSendDispatch?.();
|
||||
await params.editStreamMessage({
|
||||
laneName,
|
||||
messageId,
|
||||
@@ -383,6 +396,8 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
|
||||
durable: requestedDurable,
|
||||
allowStream = true,
|
||||
promptContextSequence: suppliedPromptContextSequence,
|
||||
onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery,
|
||||
}: DeliverLaneTextParams): Promise<LaneDeliveryResult> => {
|
||||
const lane = params.lanes[laneName];
|
||||
const promptContextSequence =
|
||||
@@ -425,6 +440,7 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
|
||||
promptContextSequence,
|
||||
false,
|
||||
streamedErrorDraftText !== undefined,
|
||||
onPlatformSendDispatch,
|
||||
)
|
||||
: undefined;
|
||||
if (streamed) {
|
||||
@@ -449,6 +465,8 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
|
||||
buttons,
|
||||
promptContextSequence,
|
||||
true,
|
||||
false,
|
||||
onPlatformSendDispatch,
|
||||
);
|
||||
if (finalizedPreview) {
|
||||
const stripButtons =
|
||||
@@ -465,6 +483,8 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
|
||||
afterAcceptedDraft: true,
|
||||
durable,
|
||||
promptContextSequence,
|
||||
onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery,
|
||||
},
|
||||
);
|
||||
return finalizedPreview;
|
||||
@@ -491,6 +511,8 @@ export function createLaneTextDeliverer(params: CreateLaneTextDelivererParams) {
|
||||
afterAcceptedDraft,
|
||||
durable,
|
||||
promptContextSequence,
|
||||
onPlatformSendDispatch,
|
||||
bindPendingFinalDelivery,
|
||||
...(retainedFinalContent?.sourceTextMode === "html" ? { textMode: "html" } : {}),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -184,6 +184,35 @@ describe("createLaneTextDeliverer", () => {
|
||||
expect(harness.lanes.answer.finalized).toBe(true);
|
||||
});
|
||||
|
||||
it("claims an equal visible preview before finalization can crash", async () => {
|
||||
const events: string[] = [];
|
||||
const answer = createTestDraftStream({ messageId: 999 });
|
||||
answer.lastDeliveredText.mockReturnValue(HELLO_FINAL);
|
||||
const harness = createHarness({ answerStream: answer });
|
||||
const finalizationCrash = new Error("injected finalization crash");
|
||||
harness.stopDraftLane.mockImplementationOnce(async () => {
|
||||
events.push("finalize");
|
||||
throw finalizationCrash;
|
||||
});
|
||||
const onPlatformSendDispatch = vi.fn(async () => {
|
||||
events.push("custody");
|
||||
});
|
||||
|
||||
await expect(
|
||||
harness.deliverLaneText({
|
||||
laneName: "answer",
|
||||
text: HELLO_FINAL,
|
||||
payload: { text: HELLO_FINAL },
|
||||
infoKind: "final",
|
||||
onPlatformSendDispatch,
|
||||
}),
|
||||
).rejects.toBe(finalizationCrash);
|
||||
|
||||
expect(events).toEqual(["custody", "finalize"]);
|
||||
expect(answer.update).not.toHaveBeenCalled();
|
||||
expect(onPlatformSendDispatch).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("streams block and final text through the same lane", async () => {
|
||||
const harness = createHarness({ answerMessageId: 999 });
|
||||
|
||||
|
||||
@@ -77,6 +77,7 @@ async function resolveTelegramSendContext(params: {
|
||||
onDeliveryResult?: Parameters<
|
||||
NonNullable<ChannelOutboundAdapter["sendText"]>
|
||||
>[0]["onDeliveryResult"];
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
resolveSend: ResolveTelegramSendFn;
|
||||
}): Promise<{
|
||||
send: TelegramSendFn;
|
||||
@@ -93,6 +94,7 @@ async function resolveTelegramSendContext(params: {
|
||||
silent?: boolean;
|
||||
gatewayClientScopes?: readonly string[];
|
||||
onDeliveryResult?: TelegramSendOpts["onDeliveryResult"];
|
||||
onPlatformSendDispatch?: TelegramSendOpts["onPlatformSendDispatch"];
|
||||
};
|
||||
}> {
|
||||
const send = await params.resolveSend(params.deps);
|
||||
@@ -113,6 +115,7 @@ async function resolveTelegramSendContext(params: {
|
||||
await params.onDeliveryResult?.(attachChannelToResult("telegram", result));
|
||||
}
|
||||
: undefined,
|
||||
onPlatformSendDispatch: params.onPlatformSendDispatch,
|
||||
...(params.formatting?.parseMode === "HTML" ? { textMode: "html" as const } : {}),
|
||||
tableMode: params.formatting?.tableMode,
|
||||
},
|
||||
@@ -383,6 +386,7 @@ export async function sendTelegramPayloadMessages(params: {
|
||||
if (typeof replyToMessageId !== "number") {
|
||||
throw new Error("Telegram reaction requires a reply target");
|
||||
}
|
||||
await params.baseOpts.onPlatformSendDispatch?.();
|
||||
const reactionResult = await params.react(params.to, replyToMessageId, reactionEmoji, {
|
||||
cfg: params.baseOpts.cfg,
|
||||
accountId: params.baseOpts.accountId,
|
||||
@@ -597,6 +601,7 @@ export function createTelegramOutboundAdapter(
|
||||
silent,
|
||||
isAnonymous,
|
||||
gatewayClientScopes,
|
||||
onPlatformSendDispatch,
|
||||
}) => {
|
||||
const outboundTo = normalizeTelegramOutboundTarget(to);
|
||||
const { sendPollTelegram } = await loadSendModule();
|
||||
@@ -607,6 +612,7 @@ export function createTelegramOutboundAdapter(
|
||||
silent: silent ?? undefined,
|
||||
isAnonymous: isAnonymous ?? undefined,
|
||||
gatewayClientScopes,
|
||||
onPlatformSendDispatch,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
@@ -72,8 +72,9 @@ async function sendLocationTelegramWithContext(
|
||||
const delivery = await withTelegramNativeQuoteFallback({
|
||||
label,
|
||||
requestParams: commonParams,
|
||||
request: (effectiveParams, retryLabel) =>
|
||||
prepared.request(
|
||||
request: async (effectiveParams, retryLabel) => {
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
return await prepared.request(
|
||||
() =>
|
||||
hasName
|
||||
? api.sendVenue(
|
||||
@@ -91,7 +92,8 @@ async function sendLocationTelegramWithContext(
|
||||
: {}),
|
||||
} as TelegramSendLocationParams),
|
||||
retryLabel,
|
||||
),
|
||||
);
|
||||
},
|
||||
});
|
||||
const result = delivery.result;
|
||||
const acceptedParams = toAcceptedThreadScopedParams(delivery.acceptedParams);
|
||||
|
||||
@@ -149,14 +149,16 @@ export function createTelegramTextSender(config: {
|
||||
withTelegramNativeQuoteFallback({
|
||||
label,
|
||||
requestParams,
|
||||
request: (effectiveParams, retryLabel) =>
|
||||
requestWithChatNotFound(
|
||||
request: async (effectiveParams, retryLabel) => {
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
return await requestWithChatNotFound(
|
||||
() =>
|
||||
Object.keys(effectiveParams).length > 0
|
||||
? api.sendMessage(chatId, messageText, effectiveParams)
|
||||
: api.sendMessage(chatId, messageText),
|
||||
retryLabel,
|
||||
),
|
||||
);
|
||||
},
|
||||
});
|
||||
const requestPlain = (label: string) =>
|
||||
requestSendMessage(label, chunk.plainText, plainParams ?? {});
|
||||
|
||||
@@ -49,6 +49,8 @@ export type TelegramSendOpts = {
|
||||
forceDocument?: boolean;
|
||||
/** Persist each concrete platform send before any later chunk can fail. */
|
||||
onDeliveryResult?: (result: TelegramSendResult) => Promise<void> | void;
|
||||
/** @internal Refresh durable custody immediately before Telegram Bot API I/O. */
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
};
|
||||
|
||||
export type TelegramApiCallOpts = Pick<
|
||||
@@ -76,5 +78,10 @@ export type TelegramSendResult = {
|
||||
export type TelegramLocationSendOpts = TelegramThreadedSendOpts &
|
||||
Pick<
|
||||
TelegramSendOpts,
|
||||
"buttons" | "quoteText" | "promptContextProjectionPlan" | "silent" | "onDeliveryResult"
|
||||
| "buttons"
|
||||
| "quoteText"
|
||||
| "promptContextProjectionPlan"
|
||||
| "silent"
|
||||
| "onDeliveryResult"
|
||||
| "onPlatformSendDispatch"
|
||||
>;
|
||||
|
||||
@@ -288,12 +288,14 @@ async function sendMessageTelegramWithContext(
|
||||
withTelegramNativeQuoteFallback({
|
||||
label,
|
||||
requestParams,
|
||||
request: (effectiveParams, effectiveLabel) =>
|
||||
requestWithChatNotFound(
|
||||
request: async (effectiveParams, effectiveLabel) => {
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
return await requestWithChatNotFound(
|
||||
() => sender(effectiveParams),
|
||||
effectiveLabel,
|
||||
shouldLog ? { shouldLog } : undefined,
|
||||
),
|
||||
);
|
||||
},
|
||||
}),
|
||||
});
|
||||
};
|
||||
|
||||
@@ -102,7 +102,7 @@ async function sendStickerTelegramWithContext(
|
||||
}
|
||||
|
||||
type TelegramPollOpts = TelegramThreadedSendOpts &
|
||||
Pick<TelegramSendOpts, "silent"> & {
|
||||
Pick<TelegramSendOpts, "onPlatformSendDispatch" | "silent"> & {
|
||||
/** Whether votes are anonymous. Defaults to true (Telegram default). */
|
||||
isAnonymous?: boolean;
|
||||
};
|
||||
@@ -163,6 +163,7 @@ async function sendPollTelegramWithContext(
|
||||
...(opts.silent === true ? { disable_notification: true } : {}),
|
||||
};
|
||||
|
||||
await opts.onPlatformSendDispatch?.();
|
||||
const result = await prepared.request(
|
||||
() =>
|
||||
api.sendPoll(prepared.chatId, normalizedPoll.question, normalizedPoll.options, pollParams),
|
||||
|
||||
@@ -828,7 +828,7 @@ describe("agentCommand compaction transcript rotation", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("skips post-turn compaction before delivering sendable finals that pending text cannot replay", async () => {
|
||||
it("compacts after persisting transport ownership for finals that text cannot replay", async () => {
|
||||
const sessionId = "unrecoverable-media-before-compaction";
|
||||
const sessionKey = `agent:main:explicit:${sessionId}`;
|
||||
const payloads = [{ mediaUrl: "/tmp/reply.ogg", audioAsVoice: true }];
|
||||
@@ -845,7 +845,7 @@ describe("agentCommand compaction transcript rotation", () => {
|
||||
deliver: true,
|
||||
});
|
||||
|
||||
expect(state.runCliTurnCompactionLifecycleMock).not.toHaveBeenCalled();
|
||||
expect(state.runCliTurnCompactionLifecycleMock).toHaveBeenCalledOnce();
|
||||
expect(state.deliverAgentCommandResultMock).toHaveBeenCalledOnce();
|
||||
expect(state.deliverAgentCommandResultMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ payloads }),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { getReplyPayloadMetadata } from "../../auto-reply/reply-payload.js";
|
||||
import type { CliDeps } from "../../cli/deps.types.js";
|
||||
import { buildRestartRecoveryClaimCleanupPatch } from "../../config/sessions/restart-recovery-state.js";
|
||||
import type { RestartRecoveryTerminalDeliveryEvidenceResult } from "../../config/sessions/restart-recovery-types.js";
|
||||
import type { SessionEntry } from "../../config/sessions/types.js";
|
||||
import { assertAgentRunLifecycleGenerationCurrent } from "../../infra/agent-events.js";
|
||||
@@ -70,6 +71,7 @@ export async function finalizeEmbeddedAgentCommand(params: {
|
||||
cwd,
|
||||
agentDir,
|
||||
outboundSession,
|
||||
runId,
|
||||
agentCfg,
|
||||
} = params.prepared;
|
||||
const {
|
||||
@@ -374,7 +376,8 @@ export async function finalizeEmbeddedAgentCommand(params: {
|
||||
!params.suppressVisibleSessionEffects &&
|
||||
!sessionReboundDuringRun
|
||||
) {
|
||||
const entry = sessionStore[sessionKey] ?? sessionEntry;
|
||||
const entry =
|
||||
(await resolveFreshSessionEntryForDelivery?.()) ?? sessionStore[sessionKey] ?? sessionEntry;
|
||||
if (!entry) {
|
||||
throw new Error("Cannot clear pending delivery without a session entry");
|
||||
}
|
||||
@@ -383,15 +386,55 @@ export async function finalizeEmbeddedAgentCommand(params: {
|
||||
params.opts.deliver === true &&
|
||||
!pendingFinalDeliveryMarker.hasSendableFinalPayload &&
|
||||
entry.pendingFinalDelivery?.kind === "transport-only";
|
||||
if (deliveryResult?.deliverySucceeded === true || clearStaleTransportOnly) {
|
||||
const clearOwnedPendingFinal =
|
||||
deliveryResult?.deliverySucceeded === true &&
|
||||
pendingFinalDeliveryMarker.pendingFinalDeliveryIntentId !== undefined;
|
||||
// Preserve the exact local claim through sibling session writes so a delivered
|
||||
// source is tombstoned before admission release can erase its ownership fields.
|
||||
const recoveryClaimEntry =
|
||||
entry.restartRecoveryDeliveryRunId === runId
|
||||
? entry
|
||||
: sessionEntry?.restartRecoveryDeliveryRunId === runId
|
||||
? sessionEntry
|
||||
: params.sessionEntry?.restartRecoveryDeliveryRunId === runId
|
||||
? params.sessionEntry
|
||||
: undefined;
|
||||
if (clearOwnedPendingFinal || clearStaleTransportOnly || recoveryClaimEntry) {
|
||||
const now = Date.now();
|
||||
sessionEntry = await persistAgentSession({
|
||||
sessionStore,
|
||||
sessionKey,
|
||||
storePath,
|
||||
initialEntry: entry,
|
||||
entry: clearPendingFinalDelivery(entry, Date.now()),
|
||||
entry: {
|
||||
...(clearOwnedPendingFinal || clearStaleTransportOnly
|
||||
? clearPendingFinalDelivery(entry, now)
|
||||
: { ...entry, updatedAt: now }),
|
||||
...(recoveryClaimEntry
|
||||
? buildRestartRecoveryClaimCleanupPatch({
|
||||
entry: {
|
||||
...recoveryClaimEntry,
|
||||
restartRecoveryTerminalDeliveryEvidence:
|
||||
entry.restartRecoveryTerminalDeliveryEvidence,
|
||||
restartRecoveryTerminalRunIds: entry.restartRecoveryTerminalRunIds,
|
||||
},
|
||||
recordTerminalSource: true,
|
||||
terminalDeliveryEvidence: buildRestartRecoveryTerminalDeliveryEvidence(
|
||||
deliveryResult ?? result,
|
||||
),
|
||||
terminalRunId: runId,
|
||||
})
|
||||
: {}),
|
||||
},
|
||||
shouldPersist: (current) =>
|
||||
shouldPersistCurrentRunSessionCleanup(current, runOwnedSessionId),
|
||||
shouldPersistCurrentRunSessionCleanup(current, runOwnedSessionId) &&
|
||||
(!recoveryClaimEntry ||
|
||||
current?.restartRecoveryDeliveryRunId === undefined ||
|
||||
current.restartRecoveryDeliveryRunId === runId) &&
|
||||
(!clearOwnedPendingFinal ||
|
||||
current?.pendingFinalDelivery?.intentId ===
|
||||
pendingFinalDeliveryMarker.pendingFinalDeliveryIntentId) &&
|
||||
(!clearStaleTransportOnly || current?.pendingFinalDelivery?.kind === "transport-only"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ export function scheduleMainSessionRecoveryPendingTarget(
|
||||
getConfig: getRuntimeConfig,
|
||||
getGatewayRuntime: getGatewayRecoveryRuntime,
|
||||
sessionKey: target.sessionKey,
|
||||
stateDir: target.stateDir,
|
||||
storePath: target.storePath,
|
||||
}),
|
||||
() => {}, // Startup recovery remains the fallback if this optional module cannot load.
|
||||
|
||||
@@ -32,6 +32,7 @@ type MainSessionRecoveryStoreResult = {
|
||||
|
||||
export type MainSessionRecoveryPendingTarget = MainSessionRecoveryStoreTarget & {
|
||||
sessionId: string;
|
||||
stateDir?: string;
|
||||
};
|
||||
|
||||
function matchesReservation(entry: SessionEntry, reservation: MainSessionRecoveryReservation) {
|
||||
|
||||
@@ -298,6 +298,7 @@ export async function markSessionCompletedAfterRecoveryCheckpoint(params: {
|
||||
agentId: string;
|
||||
entry: SessionEntry;
|
||||
messages: readonly unknown[];
|
||||
pendingFinalDeliveryIntentId?: string;
|
||||
reason: "delivered-terminal" | "delivered-terminal-receipt" | "handled-silent";
|
||||
storePath: string;
|
||||
sessionKey: string;
|
||||
@@ -321,6 +322,7 @@ export async function markSessionCompletedAfterRecoveryCheckpoint(params: {
|
||||
pendingFinalDelivery: undefined,
|
||||
restartRecoveryForceSafeTools: undefined,
|
||||
restartRecoveryRuns: undefined,
|
||||
...buildMainSessionRecoveryClearPatch(params.entry),
|
||||
runtimeMs:
|
||||
typeof params.entry.startedAt === "number"
|
||||
? Math.max(0, endedAt - params.entry.startedAt)
|
||||
@@ -471,6 +473,8 @@ export async function markSessionCompletedAfterRecoveryCheckpoint(params: {
|
||||
if (
|
||||
!entry ||
|
||||
entry.sessionId !== params.entry.sessionId ||
|
||||
(params.pendingFinalDeliveryIntentId !== undefined &&
|
||||
entry.pendingFinalDelivery?.intentId !== params.pendingFinalDeliveryIntentId) ||
|
||||
entry.status !== "running" ||
|
||||
entry.abortedLastRun !== true ||
|
||||
normalizeOptionalString(entry.restartRecoveryDeliveryRunId) !== expectedRecoveryRunId ||
|
||||
|
||||
@@ -86,6 +86,7 @@ export async function failUnresumableMainSession(params: {
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
reason: string;
|
||||
noticeText?: string;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
}): Promise<"failed" | "skipped"> {
|
||||
@@ -105,6 +106,7 @@ export async function failUnresumableMainSession(params: {
|
||||
entry: params.entry,
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
...(params.noticeText ? { text: params.noticeText } : {}),
|
||||
})) !== "written"
|
||||
) {
|
||||
// Keep ownership for another recovery attempt until its terminal notice is durable.
|
||||
@@ -126,7 +128,7 @@ export async function failUnresumableMainSession(params: {
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
reason: params.reason,
|
||||
sessionKey: params.sessionKey,
|
||||
text: UNRESUMABLE_SESSION_NOTICE,
|
||||
text: params.noticeText ?? UNRESUMABLE_SESSION_NOTICE,
|
||||
});
|
||||
}
|
||||
return "failed";
|
||||
|
||||
@@ -88,6 +88,7 @@ export async function recoverRestartAbortedMainSessions(params: {
|
||||
cfg: params.cfg,
|
||||
onExhaustedTarget: params.onExhaustedTarget,
|
||||
storePath,
|
||||
stateDir: params.stateDir,
|
||||
resumedSessionKeys,
|
||||
activeSessionIds: params.activeSessionIds,
|
||||
activeSessionKeys: params.activeSessionKeys,
|
||||
@@ -116,6 +117,7 @@ export async function retryRestartAbortedMainSessionRecovery(params: {
|
||||
expectedRecoverySourceRunId?: string;
|
||||
expectedSessionId: string;
|
||||
sessionKey: string;
|
||||
stateDir?: string;
|
||||
storePath: string;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
}): Promise<RecoveryCounts> {
|
||||
@@ -147,6 +149,7 @@ async function recoverExpectedRestartRecovery(params: {
|
||||
sessionKey: string;
|
||||
shouldContinue?: () => boolean;
|
||||
storePath: string;
|
||||
stateDir?: string;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
}): Promise<RecoveryCounts> {
|
||||
const loadExpected = () =>
|
||||
@@ -186,6 +189,7 @@ async function recoverExpectedRestartRecovery(params: {
|
||||
cfg: params.cfg,
|
||||
observationOnly: params.observationOnly,
|
||||
storePath: params.storePath,
|
||||
stateDir: params.stateDir,
|
||||
resumedSessionKeys: new Set<string>(),
|
||||
expectedClaim: params.expectedClaim,
|
||||
expectedTarget: params.expectedTarget,
|
||||
@@ -208,6 +212,7 @@ export function scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease(param
|
||||
maxRetries?: number;
|
||||
expectedSessionId: string;
|
||||
sessionKey: string;
|
||||
stateDir?: string;
|
||||
storePath: string;
|
||||
}): void {
|
||||
const recover = () =>
|
||||
@@ -220,6 +225,7 @@ export function scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease(param
|
||||
cfg: params.getConfig(),
|
||||
expectedSessionId: params.expectedSessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
stateDir: params.stateDir,
|
||||
storePath: params.storePath,
|
||||
gatewayRuntime,
|
||||
});
|
||||
@@ -327,6 +333,7 @@ export function scheduleRestartAbortedMainSessionRecovery(params: {
|
||||
sessionKey: target.sessionKey,
|
||||
shouldContinue,
|
||||
storePath: target.storePath,
|
||||
stateDir: params.stateDir,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
}),
|
||||
),
|
||||
|
||||
@@ -14,6 +14,7 @@ import type { GatewayRecoveryRuntime } from "../../gateway/server-instance-runti
|
||||
import { readSessionMessagesAsync } from "../../gateway/session-transcript-readers.js";
|
||||
import { resolveGatewaySessionStoreTarget } from "../../gateway/session-utils.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
|
||||
import { findDeliveryIntentOwner } from "../../infra/outbound/delivery-queue-storage.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
|
||||
import { resolveDefaultAgentId } from "../agent-scope-config.js";
|
||||
import {
|
||||
@@ -53,6 +54,32 @@ import {
|
||||
normalizeStringSet,
|
||||
} from "./main-session-restart-recovery-shared.js";
|
||||
|
||||
function pendingFinalRecoveryAction(
|
||||
pending: NonNullable<SessionEntry["pendingFinalDelivery"]>,
|
||||
stateDir?: string,
|
||||
): "complete" | "defer" | "fail" | "retry" {
|
||||
const deliveries = pending.deliveries;
|
||||
if (!deliveries?.length) {
|
||||
return "fail";
|
||||
}
|
||||
if (deliveries.every(({ state }) => state === "delivered" || state === "suppressed")) {
|
||||
return "complete";
|
||||
}
|
||||
const owners = deliveries.map(({ id }) => findDeliveryIntentOwner(id, stateDir));
|
||||
if (owners.some((owner) => owner?.status === "pending")) {
|
||||
return "defer";
|
||||
}
|
||||
for (const [index, delivery] of deliveries.entries()) {
|
||||
const owner = owners[index];
|
||||
if (owner || delivery.state === "delivered" || delivery.state === "unknown") {
|
||||
return "fail";
|
||||
}
|
||||
}
|
||||
return pending.kind === "replayable" && deliveries.every(({ state }) => state === "prepared")
|
||||
? "retry"
|
||||
: "fail";
|
||||
}
|
||||
|
||||
export function loadExpectedRestartRecoveryTarget(params: {
|
||||
expected: ExpectedRestartRecoveryTarget;
|
||||
storePath: string;
|
||||
@@ -99,6 +126,7 @@ export async function recoverStore(params: {
|
||||
observationOnly?: boolean;
|
||||
onExhaustedTarget?: (target: ExhaustedRestartRecoveryTarget) => void;
|
||||
storePath: string;
|
||||
stateDir?: string;
|
||||
resumedSessionKeys: Set<string>;
|
||||
expectedClaim?: ExpectedRestartRecoveryClaim;
|
||||
expectedTarget?: ExpectedRestartRecoveryTarget;
|
||||
@@ -293,7 +321,7 @@ export async function recoverStore(params: {
|
||||
}
|
||||
}
|
||||
};
|
||||
const failCurrent = async (reason: string) => {
|
||||
const failCurrent = async (reason: string, noticeText?: string) => {
|
||||
if (stopped()) {
|
||||
return false;
|
||||
}
|
||||
@@ -303,6 +331,7 @@ export async function recoverStore(params: {
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
observation: recoveryView.observation,
|
||||
reason,
|
||||
...(noticeText ? { noticeText } : {}),
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
@@ -363,6 +392,44 @@ export async function recoverStore(params: {
|
||||
);
|
||||
};
|
||||
|
||||
const pendingAction = entry.pendingFinalDelivery
|
||||
? pendingFinalRecoveryAction(entry.pendingFinalDelivery, params.stateDir)
|
||||
: undefined;
|
||||
if (pendingAction === "defer") {
|
||||
result.failed++;
|
||||
continue;
|
||||
}
|
||||
if (pendingAction === "complete") {
|
||||
const completion = await markSessionCompletedAfterRecoveryCheckpoint({
|
||||
agentId,
|
||||
entry,
|
||||
messages: [],
|
||||
pendingFinalDeliveryIntentId: entry.pendingFinalDelivery?.intentId,
|
||||
reason: "delivered-terminal-receipt",
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
if (completion.outcome === "completed") {
|
||||
params.resumedSessionKeys.add(resumeDedupeKey);
|
||||
result.recovered++;
|
||||
} else {
|
||||
result.skipped++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (pendingAction === "fail") {
|
||||
if (
|
||||
!(await failCurrent(
|
||||
"pending final delivery outcome is unknown",
|
||||
"My previous response was interrupted during delivery. " +
|
||||
"Please ask for any missing remainder; I won't rerun your previous request automatically.",
|
||||
))
|
||||
) {
|
||||
return result;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
entry.pendingFinalDelivery?.kind === "replayable" &&
|
||||
entry.restartRecoveryForceSafeTools === true
|
||||
|
||||
@@ -26,6 +26,9 @@ import {
|
||||
rotateAgentEventLifecycleGeneration,
|
||||
} from "../../infra/agent-events.js";
|
||||
import { registerAgentRunContext } from "../../infra/agent-run-registry.js";
|
||||
import { moveDeliveryQueueEntryToFailed } from "../../infra/delivery-queue-sqlite.js";
|
||||
import { OUTBOUND_DELIVERY_QUEUE_NAME } from "../../infra/outbound/delivery-queue-media-staging.js";
|
||||
import { ackDelivery, enqueueDeliveryOnce } from "../../infra/outbound/delivery-queue-storage.js";
|
||||
import {
|
||||
initializeGlobalHookRunner,
|
||||
resetGlobalHookRunner,
|
||||
@@ -262,6 +265,8 @@ function makePendingFinalDelivery(
|
||||
kind: "replayable",
|
||||
text,
|
||||
createdAt: Date.now(),
|
||||
intentId: "intent-prepared-default",
|
||||
deliveries: [{ id: "delivery-prepared-default", state: "prepared" }],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -1833,65 +1838,290 @@ describe("main-session-restart-recovery", () => {
|
||||
expect(store["agent:main:main"]?.abortedLastRun).toBe(true);
|
||||
});
|
||||
|
||||
it("resumes marked sessions with a durable pending final delivery payload (Phase 2)", async () => {
|
||||
it.each([
|
||||
["missing", undefined],
|
||||
["empty", []],
|
||||
] as const)(
|
||||
"fails closed when pending final delivery identities are %s",
|
||||
async (_, deliveries) => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
const pendingPayload = "The final answer is 42.";
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
restartRecoveryForceSafeTools: true,
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: pendingPayload,
|
||||
createdAt: Date.now() - 5_000,
|
||||
...(deliveries ? { deliveries: [...deliveries] } : {}),
|
||||
context: {
|
||||
channel: "discord",
|
||||
to: "discord:dm:final",
|
||||
accountId: "main",
|
||||
},
|
||||
},
|
||||
restartRecoveryBeforeAgentReplyState: "handled-reply",
|
||||
restartRecoveryDeliveryRunId: "discord-message-1",
|
||||
restartRecoveryDeliverySourceRunId: "discord-message-1",
|
||||
restartRecoverySourceIngress: "channel",
|
||||
restartRecoveryDeliveryContext: {
|
||||
channel: "discord",
|
||||
to: "discord:dm:stale",
|
||||
accountId: "old",
|
||||
},
|
||||
});
|
||||
await writeTranscript(sessionsDir, "main-session", [
|
||||
{ role: "user", content: "calculate the answer" },
|
||||
{ role: "assistant", content: [{ type: "toolCall", id: "call-1", name: "calc" }] },
|
||||
{ role: "toolResult", content: "42" },
|
||||
]);
|
||||
|
||||
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 }, {});
|
||||
expect(runtimePluginMocks.findRestartRecoveryUnsafeReplyHook).not.toHaveBeenCalled();
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
expect(sendRecoveryNotice).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ text: expect.stringContaining("ask for any missing remainder") }),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("retries a prepared pending final only when no queue owner exists", async () => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
const pendingPayload = "The final answer is 42.";
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
restartRecoveryForceSafeTools: true,
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: pendingPayload,
|
||||
createdAt: Date.now() - 5_000,
|
||||
context: {
|
||||
channel: "discord",
|
||||
to: "discord:dm:final",
|
||||
accountId: "main",
|
||||
},
|
||||
},
|
||||
restartRecoveryBeforeAgentReplyState: "handled-reply",
|
||||
restartRecoveryDeliveryRunId: "discord-message-1",
|
||||
restartRecoveryDeliverySourceRunId: "discord-message-1",
|
||||
restartRecoverySourceIngress: "channel",
|
||||
restartRecoveryDeliveryContext: {
|
||||
channel: "discord",
|
||||
to: "discord:dm:stale",
|
||||
accountId: "old",
|
||||
},
|
||||
pendingFinalDelivery: makePendingFinalDelivery("The prepared final answer.", {
|
||||
intentId: "intent-prepared",
|
||||
deliveries: [{ id: "delivery-prepared", state: "prepared" }],
|
||||
}),
|
||||
});
|
||||
await writeTranscript(sessionsDir, "main-session", [
|
||||
{ role: "user", content: "calculate the answer" },
|
||||
{ role: "assistant", content: [{ type: "toolCall", id: "call-1", name: "calc" }] },
|
||||
{ role: "toolResult", content: "42" },
|
||||
{ role: "user", content: "finish the answer" },
|
||||
]);
|
||||
|
||||
await expectRecovery({ recovered: 1, failed: 0, skipped: 0 }, {});
|
||||
expect(runtimePluginMocks.findRestartRecoveryUnsafeReplyHook).toHaveBeenCalledWith({
|
||||
trigger: "user",
|
||||
});
|
||||
expect(callGateway).toHaveBeenCalledOnce();
|
||||
expect(gatewayParams()).toMatchObject({
|
||||
deliver: true,
|
||||
bestEffortDeliver: true,
|
||||
channel: "discord",
|
||||
to: "discord:dm:final",
|
||||
accountId: "main",
|
||||
forceRestartSafeTools: true,
|
||||
});
|
||||
expect(gatewayParams().message).toContain(pendingPayload);
|
||||
await expectRecovery({ recovered: 1, failed: 0, skipped: 0 });
|
||||
|
||||
const beforeStoreRead = Date.now();
|
||||
const store = readStore(path.join(sessionsDir, "sessions.json"));
|
||||
const entry = store["agent:main:main"];
|
||||
expect(entry?.abortedLastRun).toBe(false);
|
||||
expect(entry?.pendingFinalDelivery).toMatchObject({
|
||||
kind: "replayable",
|
||||
text: pendingPayload,
|
||||
});
|
||||
expect(entry?.restartRecoveryForceSafeTools).toBe(true);
|
||||
expect(entry?.pendingFinalDelivery?.createdAt).toBeLessThanOrEqual(beforeStoreRead);
|
||||
expect(callGateway).toHaveBeenCalledOnce();
|
||||
expect(gatewayParams().message).toContain("The prepared final answer.");
|
||||
});
|
||||
|
||||
it("quietly completes a pending final whose deliveries are terminal", async () => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
const storePath = path.join(sessionsDir, "sessions.json");
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
pendingFinalDelivery: makePendingFinalDelivery("Already delivered.", {
|
||||
intentId: "intent-delivered",
|
||||
deliveries: [
|
||||
{ id: "delivery-delivered", state: "delivered" },
|
||||
{ id: "delivery-suppressed", state: "suppressed" },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
await expectRecovery({ recovered: 1, failed: 0, skipped: 0 });
|
||||
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
expect(sendRecoveryNotice).not.toHaveBeenCalled();
|
||||
expect(loadSessionEntry({ sessionKey: "agent:main:main", storePath })).toMatchObject({
|
||||
status: "done",
|
||||
abortedLastRun: false,
|
||||
});
|
||||
expect(
|
||||
loadSessionEntry({ sessionKey: "agent:main:main", storePath })?.pendingFinalDelivery,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
{ id: "delivery-already-delivered", state: "delivered" as const },
|
||||
{ id: "delivery-still-pending", state: "queued" as const },
|
||||
],
|
||||
[
|
||||
{ id: "delivery-still-pending", state: "queued" as const },
|
||||
{ id: "delivery-already-delivered", state: "delivered" as const },
|
||||
],
|
||||
])("defers mixed deliveries while any exact queue owner is pending", async (...deliveries) => {
|
||||
try {
|
||||
await enqueueDeliveryOnce(
|
||||
{
|
||||
channel: "discord",
|
||||
to: "discord:dm:123",
|
||||
payloads: [{ text: "Pending sibling." }],
|
||||
queuePolicy: "required",
|
||||
completionRetention: "permanent",
|
||||
},
|
||||
"delivery-still-pending",
|
||||
tmpDir,
|
||||
);
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
pendingFinalDelivery: makePendingFinalDelivery("Partially delivered answer.", {
|
||||
context: discordDeliveryContext,
|
||||
intentId: "intent-mixed-pending",
|
||||
deliveries,
|
||||
}),
|
||||
});
|
||||
|
||||
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
|
||||
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
expect(sendRecoveryNotice).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
}
|
||||
});
|
||||
|
||||
it("completes terminal deliveries despite a residual pending queue row", async () => {
|
||||
try {
|
||||
await enqueueDeliveryOnce(
|
||||
{
|
||||
channel: "discord",
|
||||
to: "discord:dm:123",
|
||||
payloads: [{ text: "Already delivered." }],
|
||||
queuePolicy: "required",
|
||||
completionRetention: "permanent",
|
||||
},
|
||||
"delivery-terminal-with-row",
|
||||
tmpDir,
|
||||
);
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
const storePath = path.join(sessionsDir, "sessions.json");
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
pendingFinalDelivery: makePendingFinalDelivery("Already delivered.", {
|
||||
intentId: "intent-terminal-with-row",
|
||||
deliveries: [{ id: "delivery-terminal-with-row", state: "delivered" }],
|
||||
}),
|
||||
});
|
||||
|
||||
await expectRecovery({ recovered: 1, failed: 0, skipped: 0 });
|
||||
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
expect(sendRecoveryNotice).not.toHaveBeenCalled();
|
||||
expect(loadSessionEntry({ sessionKey: "agent:main:main", storePath })?.status).toBe("done");
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed for an unqueued media-only final", async () => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
pendingFinalDelivery: {
|
||||
kind: "transport-only",
|
||||
createdAt: Date.now(),
|
||||
intentId: "intent-media-only",
|
||||
deliveries: [{ id: "delivery-media-only", state: "prepared" }],
|
||||
context: discordDeliveryContext,
|
||||
},
|
||||
});
|
||||
|
||||
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
|
||||
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
expect(sendRecoveryNotice).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ text: expect.stringContaining("ask for any missing remainder") }),
|
||||
);
|
||||
});
|
||||
|
||||
it("fails visibly instead of replaying part of an unqueued text and media final", async () => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
pendingFinalDelivery: {
|
||||
kind: "transport-only",
|
||||
createdAt: Date.now(),
|
||||
context: discordDeliveryContext,
|
||||
intentId: "intent-text-media",
|
||||
deliveries: [
|
||||
{ id: "delivery-text", state: "prepared" },
|
||||
{ id: "delivery-media", state: "prepared" },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
|
||||
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
expect(sendRecoveryNotice).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each(["delivered", "unknown"] as const)(
|
||||
"fails closed when a %s delivery is mixed with prepared work",
|
||||
async (state) => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
pendingFinalDelivery: makePendingFinalDelivery("Do not regenerate this aggregate.", {
|
||||
context: discordDeliveryContext,
|
||||
intentId: `intent-mixed-${state}`,
|
||||
deliveries: [
|
||||
{ id: `delivery-${state}`, state },
|
||||
{ id: "delivery-still-prepared", state: "prepared" },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
|
||||
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
expect(sendRecoveryNotice).toHaveBeenCalledOnce();
|
||||
expect(sendRecoveryNotice).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
text: expect.stringContaining("ask for any missing remainder"),
|
||||
}),
|
||||
);
|
||||
expect(sendRecoveryNotice.mock.calls[0]?.[0].text).not.toContain("send that last request");
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["pending", "failed", "completed"] as const)(
|
||||
"does not regenerate a prepared pending final while its exact queue owner is %s",
|
||||
async (ownerStatus) => {
|
||||
const deliveryId = `delivery-owner-${ownerStatus}`;
|
||||
try {
|
||||
await enqueueDeliveryOnce(
|
||||
{
|
||||
channel: "discord",
|
||||
to: "discord:dm:123",
|
||||
payloads: [{ text: "Queue owns this final." }],
|
||||
queuePolicy: "required",
|
||||
completionRetention: "permanent",
|
||||
},
|
||||
deliveryId,
|
||||
tmpDir,
|
||||
);
|
||||
if (ownerStatus === "failed") {
|
||||
moveDeliveryQueueEntryToFailed(OUTBOUND_DELIVERY_QUEUE_NAME, deliveryId, tmpDir);
|
||||
} else if (ownerStatus === "completed") {
|
||||
await ackDelivery(deliveryId, tmpDir);
|
||||
}
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
pendingFinalDelivery: makePendingFinalDelivery("Queue owns this final.", {
|
||||
context: discordDeliveryContext,
|
||||
intentId: `intent-owner-${ownerStatus}`,
|
||||
deliveries: [{ id: deliveryId, state: "prepared" }],
|
||||
}),
|
||||
});
|
||||
|
||||
await expectRecovery({ recovered: 0, failed: 1, skipped: 0 });
|
||||
|
||||
expect(callGateway).not.toHaveBeenCalled();
|
||||
if (ownerStatus === "pending") {
|
||||
expect(sendRecoveryNotice).not.toHaveBeenCalled();
|
||||
} else {
|
||||
expect(sendRecoveryNotice).toHaveBeenCalledOnce();
|
||||
}
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a hook-owned pending final behind the unsafe-hook gate after claim cleanup", async () => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
const storePath = path.join(sessionsDir, "sessions.json");
|
||||
@@ -1964,11 +2194,9 @@ describe("main-session-restart-recovery", () => {
|
||||
].join("\n");
|
||||
await writeMainSession({
|
||||
sessionsDir,
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: pendingPayload,
|
||||
pendingFinalDelivery: makePendingFinalDelivery(pendingPayload, {
|
||||
createdAt: Date.now() - 5_000,
|
||||
},
|
||||
}),
|
||||
});
|
||||
await writeTranscript(sessionsDir, "main-session", [
|
||||
{ role: "user", content: "calculate the answer" },
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getReplyPayloadMetadata } from "../auto-reply/reply-payload.js";
|
||||
import type { SessionEntry } from "../config/sessions/types.js";
|
||||
import { persistPendingFinalDeliveryMarker } from "./pending-final-delivery-marker.js";
|
||||
|
||||
const state = vi.hoisted(() => ({ persistAgentSession: vi.fn() }));
|
||||
|
||||
vi.mock("./command/attempt-execution.shared.js", () => ({
|
||||
persistAgentSession: (...args: unknown[]) => state.persistAgentSession(...args),
|
||||
}));
|
||||
|
||||
describe("persistPendingFinalDeliveryMarker", () => {
|
||||
beforeEach(() => {
|
||||
state.persistAgentSession
|
||||
.mockReset()
|
||||
.mockImplementation(async (params: { entry: SessionEntry }) => params.entry);
|
||||
});
|
||||
|
||||
it("owns a multi-payload command delivery as one durable batch", async () => {
|
||||
const entry: SessionEntry = { sessionId: "session-1", updatedAt: 1 };
|
||||
const payloads = [{ text: "first" }, { text: "second" }];
|
||||
|
||||
const result = await persistPendingFinalDeliveryMarker({
|
||||
deliver: true,
|
||||
sessionStore: { main: entry },
|
||||
sessionKey: "main",
|
||||
sessionEntry: entry,
|
||||
storePath: "/tmp/sessions.json",
|
||||
suppressVisibleSessionEffects: false,
|
||||
sessionReboundDuringRun: false,
|
||||
payloads,
|
||||
deliveryContext: { channel: "discord", to: "channel:c1" },
|
||||
runOwnedSessionId: "session-1",
|
||||
});
|
||||
|
||||
expect(result.sessionEntry?.pendingFinalDelivery?.deliveries).toEqual([
|
||||
{ id: expect.any(String), state: "prepared" },
|
||||
]);
|
||||
const deliveryId = result.sessionEntry?.pendingFinalDelivery?.deliveries?.[0]?.id;
|
||||
expect(
|
||||
payloads.map(
|
||||
(payload) => getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion?.deliveryId,
|
||||
),
|
||||
).toEqual([deliveryId, deliveryId]);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
/** Persists restart-recoverable final delivery markers for agent runs. */
|
||||
import type { ReplyPayload } from "../auto-reply/reply-payload.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { setReplyPayloadMetadata, type ReplyPayload } from "../auto-reply/reply-payload.js";
|
||||
import {
|
||||
buildRecoverablePendingFinalDeliveryText,
|
||||
normalizePendingFinalDeliveryPayloads,
|
||||
@@ -26,15 +27,20 @@ type PersistPendingFinalDeliveryMarkerParams = {
|
||||
type PendingFinalDeliveryMarkerResult = {
|
||||
sessionEntry?: SessionEntry;
|
||||
pendingFinalDeliveryMarkerPersisted: boolean;
|
||||
pendingFinalDeliveryIntentId?: string;
|
||||
hasSendableFinalPayload: boolean;
|
||||
};
|
||||
|
||||
export async function persistPendingFinalDeliveryMarker(
|
||||
params: PersistPendingFinalDeliveryMarkerParams,
|
||||
): Promise<PendingFinalDeliveryMarkerResult> {
|
||||
const recoveryPayloads = normalizePendingFinalRecoveryPayloads(params.payloads);
|
||||
const hasSendableFinalPayload = normalizePendingFinalDeliveryPayloads(params.payloads).length > 0;
|
||||
const recoverableText = buildRecoverablePendingFinalDeliveryText(recoveryPayloads);
|
||||
const sendablePayloads = params.payloads.filter(
|
||||
(payload) => normalizePendingFinalDeliveryPayloads([payload]).length > 0,
|
||||
);
|
||||
const hasSendableFinalPayload = sendablePayloads.length > 0;
|
||||
const recoverableText = buildRecoverablePendingFinalDeliveryText(
|
||||
normalizePendingFinalRecoveryPayloads(params.payloads),
|
||||
);
|
||||
|
||||
if (
|
||||
!params.deliver ||
|
||||
@@ -42,10 +48,10 @@ export async function persistPendingFinalDeliveryMarker(
|
||||
!params.sessionKey ||
|
||||
params.suppressVisibleSessionEffects ||
|
||||
params.sessionReboundDuringRun ||
|
||||
params.payloads.length === 0 ||
|
||||
isSubagentSessionKey(params.sessionKey) ||
|
||||
!recoverableText ||
|
||||
!hasSendableFinalPayload ||
|
||||
// A run without a resolvable delivery route (e.g. rejected best-effort
|
||||
// target) must not leave a custody marker restart recovery could act on.
|
||||
!params.deliveryContext
|
||||
) {
|
||||
return {
|
||||
@@ -65,6 +71,8 @@ export async function persistPendingFinalDeliveryMarker(
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const intentId = randomUUID();
|
||||
const deliveryId = randomUUID();
|
||||
const persisted = await persistAgentSession({
|
||||
sessionStore: params.sessionStore,
|
||||
sessionKey: params.sessionKey,
|
||||
@@ -73,8 +81,11 @@ export async function persistPendingFinalDeliveryMarker(
|
||||
entry: {
|
||||
...entry,
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: recoverableText,
|
||||
...(recoverableText
|
||||
? { kind: "replayable" as const, text: recoverableText }
|
||||
: { kind: "transport-only" as const }),
|
||||
intentId,
|
||||
deliveries: [{ id: deliveryId, state: "prepared" as const }],
|
||||
createdAt: now,
|
||||
context: params.deliveryContext,
|
||||
},
|
||||
@@ -83,13 +94,29 @@ export async function persistPendingFinalDeliveryMarker(
|
||||
shouldPersist: (current) =>
|
||||
current?.sessionId === params.runOwnedSessionId && current.abortedLastRun !== true,
|
||||
});
|
||||
const markerPersisted =
|
||||
persisted?.pendingFinalDelivery?.kind === "replayable" &&
|
||||
persisted.pendingFinalDelivery.text === recoverableText;
|
||||
const markerPersisted = persisted?.pendingFinalDelivery?.intentId === intentId;
|
||||
|
||||
if (markerPersisted) {
|
||||
for (const payload of sendablePayloads) {
|
||||
setReplyPayloadMetadata(payload, {
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId,
|
||||
intentId,
|
||||
...(entry.restartRecoveryDeliveryRunId
|
||||
? { recoveryRunId: entry.restartRecoveryDeliveryRunId }
|
||||
: {}),
|
||||
sessionId: params.runOwnedSessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
sessionEntry: persisted,
|
||||
pendingFinalDeliveryMarkerPersisted: markerPersisted,
|
||||
...(markerPersisted ? { pendingFinalDeliveryIntentId: intentId } : {}),
|
||||
hasSendableFinalPayload,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -245,10 +245,15 @@ export type ReplyPayloadMetadata = {
|
||||
};
|
||||
/** Opaque owner for one final-delivery transcript capture on a shared dispatcher. */
|
||||
finalDeliveryCapture?: object;
|
||||
/** Durable pending-final intent represented by this runtime payload. */
|
||||
pendingFinalDeliveryIntentId?: string;
|
||||
/** Restart-safe text this payload contributes to its pending-final intent. */
|
||||
pendingFinalDeliveryRetryText?: string;
|
||||
/** Exact persisted delivery owner; WeakMap-only and never serialized. */
|
||||
pendingFinalDeliveryCompletion?: {
|
||||
deliveryId: string;
|
||||
intentId: string;
|
||||
recoveryRunId?: string;
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
};
|
||||
/** replyToId existed before reply threading could inject an implicit target. */
|
||||
replyToIdExplicit?: boolean;
|
||||
/** Canonical reply policy used by both message-tool dedupe and final delivery routing. */
|
||||
|
||||
@@ -19,7 +19,6 @@ import {
|
||||
normalizeDeliveryContext,
|
||||
} from "../../utils/delivery-context.shared.js";
|
||||
import { resolveFallbackTransition } from "../fallback-state.js";
|
||||
import { stripHeartbeatToken } from "../heartbeat.js";
|
||||
import {
|
||||
isReplyPayloadStatusNotice,
|
||||
markReplyPayloadForSourceSuppressionDelivery,
|
||||
@@ -37,10 +36,7 @@ import type { BlockReplyPipeline } from "./block-reply-pipeline.js";
|
||||
import { resolveEffectiveReplyRoute } from "./effective-reply-route.js";
|
||||
import type { InternalGetReplyOptions } from "./get-reply.types.js";
|
||||
import { normalizeReplyPayload } from "./normalize-reply.js";
|
||||
import {
|
||||
buildPendingFinalDeliveryText,
|
||||
sanitizePendingFinalDeliveryText,
|
||||
} from "./pending-final-delivery.js";
|
||||
import { sanitizePendingFinalDeliveryText } from "./pending-final-delivery.js";
|
||||
import { type FollowupRun, type QueueSettings, scheduleFollowupDrain } from "./queue.js";
|
||||
import { normalizeReplyPayloadDirectives } from "./reply-delivery.js";
|
||||
import { type ReplyOperation, runAfterReplyOperationClear } from "./reply-run-registry.js";
|
||||
@@ -77,18 +73,6 @@ export function markBeforeAgentRunBlockedPayloads(payloads: ReplyPayload[]): Rep
|
||||
);
|
||||
}
|
||||
|
||||
export function resolvePendingFinalDeliveryRetryText(params: {
|
||||
isHeartbeat: boolean;
|
||||
payload: ReplyPayload;
|
||||
}): string {
|
||||
const pendingText = buildPendingFinalDeliveryText([params.payload]);
|
||||
if (!params.isHeartbeat) {
|
||||
return pendingText;
|
||||
}
|
||||
const stripped = stripHeartbeatToken(pendingText, { mode: "message" });
|
||||
return stripped.shouldSkip ? "" : stripped.text || pendingText;
|
||||
}
|
||||
|
||||
export function buildSilentFallbackFailurePayload(params: {
|
||||
fallbackTransition: ReturnType<typeof resolveFallbackTransition>;
|
||||
fallbackFailureKnown: boolean;
|
||||
|
||||
@@ -319,15 +319,25 @@ export async function executePreparedReplyAgentRun(
|
||||
});
|
||||
if (!sourceReplyPolicy.suppressDelivery) {
|
||||
const pendingFinalDeliveryIntentId = crypto.randomUUID();
|
||||
const pendingFinalDeliveryDeliveryId = crypto.randomUUID();
|
||||
setReplyPayloadMetadata(hookReply, {
|
||||
pendingFinalDeliveryIntentId,
|
||||
pendingFinalDeliveryRetryText: hookFinalDeliveryText,
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId: pendingFinalDeliveryDeliveryId,
|
||||
intentId: pendingFinalDeliveryIntentId,
|
||||
...(activeSessionEntry?.restartRecoveryDeliveryRunId
|
||||
? { recoveryRunId: activeSessionEntry.restartRecoveryDeliveryRunId }
|
||||
: {}),
|
||||
sessionId: replyOperation.sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
hookCheckpoint = {
|
||||
state: hookFinalDeliveryText ? "handled-reply" : "handled-unrecoverable",
|
||||
pendingFinalDelivery: {
|
||||
text: hookFinalDeliveryText ?? "",
|
||||
intentId: pendingFinalDeliveryIntentId,
|
||||
deliveries: [{ id: pendingFinalDeliveryDeliveryId, state: "prepared" }],
|
||||
context: resolveReplyRunDeliveryContext({
|
||||
cfg,
|
||||
sessionCtx,
|
||||
|
||||
@@ -14,7 +14,6 @@ import type { ReplyPayload } from "../types.js";
|
||||
import {
|
||||
buildInlinePluginStatusPayload,
|
||||
markBeforeAgentRunBlockedPayloads,
|
||||
resolvePendingFinalDeliveryRetryText,
|
||||
resolveReplyRunDeliveryContext,
|
||||
resolveSourceReplyPolicy,
|
||||
} from "./agent-runner-core.js";
|
||||
@@ -35,7 +34,11 @@ import {
|
||||
mergeExecutionTrace,
|
||||
} from "./agent-runner-trace.js";
|
||||
import { appendUsageLine } from "./agent-runner-usage-line.js";
|
||||
import { buildPendingFinalDeliveryText } from "./pending-final-delivery.js";
|
||||
import {
|
||||
buildRecoverablePendingFinalDeliveryText,
|
||||
normalizePendingFinalDeliveryPayloads,
|
||||
normalizePendingFinalRecoveryPayloads,
|
||||
} from "./pending-final-delivery.js";
|
||||
import { readPostCompactionContext } from "./post-compaction-context.js";
|
||||
import { warnPrivateMessageToolFinal } from "./private-message-tool-final.js";
|
||||
import { enqueueFollowupRun, refreshQueuedFollowupSession } from "./queue.js";
|
||||
@@ -310,10 +313,9 @@ export async function completeReplyAgentRun(input: {
|
||||
runtimePolicySessionKey,
|
||||
opts,
|
||||
});
|
||||
const finalDeliveryText = buildPendingFinalDeliveryText(finalPayloads);
|
||||
// #85714: warn only for unusually substantive private final text. In
|
||||
// message_tool_only, no tool call can be intentional silence, and
|
||||
// finalDeliveryText also includes verbose/status/usage metadata.
|
||||
// final payloads also include verbose/status/usage metadata.
|
||||
const assistantFinalText = normalizeAssistantFinalDeliveryText(
|
||||
typeof runResult.meta?.finalAssistantVisibleText === "string"
|
||||
? runResult.meta.finalAssistantVisibleText
|
||||
@@ -357,7 +359,12 @@ export async function completeReplyAgentRun(input: {
|
||||
finalPayloads = [...finalPayloads, buildStrandedReplyDeliveryFailurePayload()];
|
||||
}
|
||||
}
|
||||
const pendingText = sourceReplyPolicy.suppressDelivery ? "" : finalDeliveryText;
|
||||
const recoverablePendingFinalText = buildRecoverablePendingFinalDeliveryText(
|
||||
normalizePendingFinalRecoveryPayloads(finalPayloads),
|
||||
);
|
||||
const pendingText = sourceReplyPolicy.suppressDelivery
|
||||
? ""
|
||||
: (recoverablePendingFinalText ?? "");
|
||||
const heartbeatAckMaxChars = DEFAULT_HEARTBEAT_ACK_MAX_CHARS;
|
||||
const resolvedPendingText = isHeartbeat
|
||||
? (() => {
|
||||
@@ -368,17 +375,30 @@ export async function completeReplyAgentRun(input: {
|
||||
return stripped.shouldSkip ? "" : stripped.text || pendingText;
|
||||
})()
|
||||
: pendingText;
|
||||
if (resolvedPendingText) {
|
||||
const sendableFinalPayloads = sourceReplyPolicy.suppressDelivery
|
||||
? []
|
||||
: finalPayloads.filter(
|
||||
(payload) => normalizePendingFinalDeliveryPayloads([payload]).length > 0,
|
||||
);
|
||||
if (sendableFinalPayloads.length > 0) {
|
||||
const pendingFinalDeliveryIntentId = crypto.randomUUID();
|
||||
for (const payload of finalPayloads) {
|
||||
const expectedSessionId = activeSessionEntry?.sessionId ?? followupRun.run.sessionId;
|
||||
const pendingFinalDeliveries = sendableFinalPayloads.map((payload) => {
|
||||
const deliveryId = crypto.randomUUID();
|
||||
setReplyPayloadMetadata(payload, {
|
||||
pendingFinalDeliveryIntentId,
|
||||
pendingFinalDeliveryRetryText: resolvePendingFinalDeliveryRetryText({
|
||||
isHeartbeat,
|
||||
payload,
|
||||
}),
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId,
|
||||
intentId: pendingFinalDeliveryIntentId,
|
||||
...(activeSessionEntry?.restartRecoveryDeliveryRunId
|
||||
? { recoveryRunId: activeSessionEntry.restartRecoveryDeliveryRunId }
|
||||
: {}),
|
||||
sessionId: expectedSessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
}
|
||||
return { id: deliveryId, state: "prepared" as const };
|
||||
});
|
||||
const pendingFinalDeliveryContext = resolveReplyRunDeliveryContext({
|
||||
cfg,
|
||||
sessionCtx,
|
||||
@@ -387,7 +407,6 @@ export async function completeReplyAgentRun(input: {
|
||||
runtimePolicySessionKey,
|
||||
opts,
|
||||
});
|
||||
const expectedSessionId = activeSessionEntry?.sessionId ?? followupRun.run.sessionId;
|
||||
// A reset can rebind the key while the model runs; its replacement must
|
||||
// never inherit the old run's final or advertise an uncommitted intent.
|
||||
const persistedPendingFinalDelivery = await updateSessionEntry(
|
||||
@@ -396,9 +415,11 @@ export async function completeReplyAgentRun(input: {
|
||||
entry.sessionId === expectedSessionId
|
||||
? {
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable" as const,
|
||||
text: resolvedPendingText,
|
||||
...(resolvedPendingText
|
||||
? { kind: "replayable" as const, text: resolvedPendingText }
|
||||
: { kind: "transport-only" as const }),
|
||||
intentId: pendingFinalDeliveryIntentId,
|
||||
deliveries: pendingFinalDeliveries,
|
||||
context: pendingFinalDeliveryContext,
|
||||
createdAt: Date.now(),
|
||||
},
|
||||
|
||||
@@ -1588,16 +1588,84 @@ describe("runReplyAgent pending final delivery capture", () => {
|
||||
kind: "replayable",
|
||||
text: "visible final",
|
||||
intentId: expect.any(String),
|
||||
deliveries: [{ id: expect.any(String), state: "prepared" }],
|
||||
});
|
||||
const visiblePayload = (Array.isArray(result) ? result : [result]).find(
|
||||
(payload) => payload?.text === "visible final",
|
||||
);
|
||||
expect(getReplyPayloadMetadata(visiblePayload ?? {})).toMatchObject({
|
||||
pendingFinalDeliveryIntentId: stored.pendingFinalDelivery?.intentId,
|
||||
pendingFinalDeliveryRetryText: "visible final",
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId: stored.pendingFinalDelivery?.deliveries?.[0]?.id,
|
||||
intentId: stored.pendingFinalDelivery?.intentId,
|
||||
sessionId: "session",
|
||||
sessionKey: "main",
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("owns a media-only final with its complete replay directive", async () => {
|
||||
const { sessionEntry, sessionStore, storePath } = await makeSessionFixture();
|
||||
state.runEmbeddedAgentMock.mockResolvedValueOnce({
|
||||
payloads: [{ mediaUrl: "https://example.test/final.png" }],
|
||||
meta: {},
|
||||
});
|
||||
const { run } = createMinimalRun({
|
||||
sessionEntry,
|
||||
sessionStore,
|
||||
sessionKey: "main",
|
||||
storePath,
|
||||
});
|
||||
|
||||
const result = await run();
|
||||
const stored = await readStoredMainSession(storePath);
|
||||
expect(stored.pendingFinalDelivery).toMatchObject({
|
||||
kind: "replayable",
|
||||
text: "MEDIA:https://example.test/final.png",
|
||||
intentId: expect.any(String),
|
||||
deliveries: [{ id: expect.any(String), state: "prepared" }],
|
||||
});
|
||||
const payload = Array.isArray(result) ? result[0] : result;
|
||||
expect(getReplyPayloadMetadata(payload ?? {})).toMatchObject({
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId: stored.pendingFinalDelivery?.deliveries?.[0]?.id,
|
||||
intentId: stored.pendingFinalDelivery?.intentId,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("owns mixed text and media finals without replaying a partial aggregate", async () => {
|
||||
const { sessionEntry, sessionStore, storePath } = await makeSessionFixture();
|
||||
state.runEmbeddedAgentMock.mockResolvedValueOnce({
|
||||
payloads: [{ text: "visible text" }, { mediaUrl: "https://example.test/final.png" }],
|
||||
meta: {},
|
||||
});
|
||||
const { run } = createMinimalRun({
|
||||
sessionEntry,
|
||||
sessionStore,
|
||||
sessionKey: "main",
|
||||
storePath,
|
||||
});
|
||||
|
||||
const result = await run();
|
||||
const payloads = Array.isArray(result) ? result : [result];
|
||||
const stored = await readStoredMainSession(storePath);
|
||||
expect(stored.pendingFinalDelivery).toMatchObject({
|
||||
kind: "transport-only",
|
||||
deliveries: [
|
||||
{ id: expect.any(String), state: "prepared" },
|
||||
{ id: expect.any(String), state: "prepared" },
|
||||
],
|
||||
});
|
||||
expect(stored.pendingFinalDelivery).not.toHaveProperty("text");
|
||||
expect(
|
||||
payloads.map(
|
||||
(payload) =>
|
||||
getReplyPayloadMetadata(payload ?? {})?.pendingFinalDeliveryCompletion?.deliveryId,
|
||||
),
|
||||
).toEqual(stored.pendingFinalDelivery?.deliveries?.map(({ id }) => id));
|
||||
});
|
||||
|
||||
it("persists canonical SQLite pending final delivery after its intent commits", async () => {
|
||||
const sessionKey = "agent:main:main";
|
||||
const { sessionEntry, sessionStore, storePath } = await makeSessionFixture({}, sessionKey);
|
||||
@@ -1620,13 +1688,19 @@ describe("runReplyAgent pending final delivery capture", () => {
|
||||
kind: "replayable",
|
||||
intentId: expect.any(String),
|
||||
text: "visible canonical final",
|
||||
deliveries: [{ id: expect.any(String), state: "prepared" }],
|
||||
},
|
||||
sessionId: "session",
|
||||
});
|
||||
const visiblePayload = Array.isArray(result) ? result[0] : result;
|
||||
expect(getReplyPayloadMetadata(visiblePayload ?? {})).toMatchObject({
|
||||
pendingFinalDeliveryIntentId: stored?.pendingFinalDelivery?.intentId,
|
||||
pendingFinalDeliveryRetryText: "visible canonical final",
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId: stored?.pendingFinalDelivery?.deliveries?.[0]?.id,
|
||||
intentId: stored?.pendingFinalDelivery?.intentId,
|
||||
sessionId: "session",
|
||||
sessionKey,
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
expect(state.runEmbeddedAgentMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
@@ -2857,11 +2931,17 @@ describe("runReplyAgent pending final delivery capture", () => {
|
||||
expect(stored.pendingFinalDelivery).toMatchObject({
|
||||
kind: "replayable",
|
||||
text: longRemainder,
|
||||
deliveries: [{ id: expect.any(String), state: "prepared" }],
|
||||
});
|
||||
const payload = Array.isArray(result) ? result[0] : result;
|
||||
expect(getReplyPayloadMetadata(payload ?? {})).toMatchObject({
|
||||
pendingFinalDeliveryIntentId: stored.pendingFinalDelivery?.intentId,
|
||||
pendingFinalDeliveryRetryText: longRemainder,
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId: stored.pendingFinalDelivery?.deliveries?.[0]?.id,
|
||||
intentId: stored.pendingFinalDelivery?.intentId,
|
||||
sessionId: "session",
|
||||
sessionKey: "main",
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
// Tests before-deliver hook ordering and payload mutation behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createDeferred } from "../../../test/helpers/promise.js";
|
||||
import { createDirectPendingFinalCustody } from "../../channels/turn/direct-delivery-custody.js";
|
||||
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import type { InternalSessionEntry } from "../../config/sessions/types.js";
|
||||
import { getReplyPayloadMetadata, setReplyPayloadMetadata } from "../reply-payload.js";
|
||||
import type { ReplyPayload } from "../types.js";
|
||||
import {
|
||||
@@ -9,6 +16,40 @@ import {
|
||||
createReplyDispatcher,
|
||||
} from "./reply-dispatcher.js";
|
||||
|
||||
async function makePendingFinalFixture() {
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-dispatcher-pending-final-"));
|
||||
const storePath = path.join(tmpDir, "sessions.json");
|
||||
const sessionKey = "agent:main:telegram:direct:123";
|
||||
await replaceSessionEntry(
|
||||
{ sessionKey, storePath },
|
||||
{
|
||||
sessionId: "session-1",
|
||||
status: "running",
|
||||
updatedAt: Date.now(),
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: "final answer",
|
||||
createdAt: Date.now(),
|
||||
intentId: "intent-1",
|
||||
deliveries: [{ id: "delivery-1", state: "prepared" }],
|
||||
},
|
||||
},
|
||||
);
|
||||
const payload = setReplyPayloadMetadata(
|
||||
{ text: "final answer" },
|
||||
{
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId: "delivery-1",
|
||||
intentId: "intent-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
storePath,
|
||||
},
|
||||
},
|
||||
);
|
||||
return { payload, sessionKey, storePath, tmpDir };
|
||||
}
|
||||
|
||||
describe("beforeDeliver in reply dispatcher", () => {
|
||||
it("delivers the attached fallback when the primary payload is cancelled", async () => {
|
||||
const delivered: string[] = [];
|
||||
@@ -301,4 +342,201 @@ describe("beforeDeliver in reply dispatcher", () => {
|
||||
|
||||
expect(delivered).toEqual(["plain reply"]);
|
||||
});
|
||||
|
||||
it("records direct-delivery custody before waiting for the channel provider", async () => {
|
||||
const fixture = await makePendingFinalFixture();
|
||||
const enteredProvider = createDeferred();
|
||||
const releaseProvider = createDeferred();
|
||||
try {
|
||||
const dispatcher = createReplyDispatcher({
|
||||
deliver: async () => {
|
||||
enteredProvider.resolve();
|
||||
await releaseProvider.promise;
|
||||
},
|
||||
});
|
||||
|
||||
dispatcher.sendFinalReply(fixture.payload);
|
||||
dispatcher.markComplete();
|
||||
await enteredProvider.promise;
|
||||
|
||||
expect(
|
||||
(
|
||||
loadSessionEntry({
|
||||
sessionKey: fixture.sessionKey,
|
||||
storePath: fixture.storePath,
|
||||
}) as InternalSessionEntry
|
||||
)?.pendingFinalDelivery?.deliveries,
|
||||
).toEqual([{ id: "delivery-1", state: "queued" }]);
|
||||
|
||||
releaseProvider.resolve();
|
||||
await dispatcher.waitForIdle();
|
||||
expect(
|
||||
(
|
||||
loadSessionEntry({
|
||||
sessionKey: fixture.sessionKey,
|
||||
storePath: fixture.storePath,
|
||||
}) as InternalSessionEntry
|
||||
)?.pendingFinalDelivery?.deliveries,
|
||||
).toEqual([{ id: "delivery-1", state: "delivered" }]);
|
||||
} finally {
|
||||
releaseProvider.resolve();
|
||||
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "proven pre-send failure",
|
||||
error: () =>
|
||||
Object.assign(new Error("connect failed"), { code: "ECONNREFUSED", syscall: "connect" }),
|
||||
expected: "prepared",
|
||||
},
|
||||
{
|
||||
label: "ambiguous provider failure",
|
||||
error: () => new Error("send outcome unknown"),
|
||||
expected: "unknown",
|
||||
},
|
||||
] as const)("records $label before reporting the error", async ({ error, expected }) => {
|
||||
const fixture = await makePendingFinalFixture();
|
||||
try {
|
||||
const dispatcher = createReplyDispatcher({
|
||||
deliver: async () => {
|
||||
throw error();
|
||||
},
|
||||
});
|
||||
|
||||
dispatcher.sendFinalReply(fixture.payload);
|
||||
dispatcher.markComplete();
|
||||
await dispatcher.waitForIdle();
|
||||
|
||||
expect(
|
||||
(
|
||||
loadSessionEntry({
|
||||
sessionKey: fixture.sessionKey,
|
||||
storePath: fixture.storePath,
|
||||
}) as InternalSessionEntry
|
||||
)?.pendingFinalDelivery?.deliveries,
|
||||
).toEqual([{ id: "delivery-1", state: expected }]);
|
||||
} finally {
|
||||
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("restores prepared custody when a pre-I/O admitted send proves no-send", async () => {
|
||||
const fixture = await makePendingFinalFixture();
|
||||
try {
|
||||
const dispatcher = createReplyDispatcher({
|
||||
deliver: async (payload) => {
|
||||
// Mirror the channel-turn direct path: custody escalates queued→unknown
|
||||
// immediately before wire I/O, then the provider proves no send happened.
|
||||
const custody = createDirectPendingFinalCustody(payload);
|
||||
await custody?.onPlatformSendDispatch();
|
||||
throw Object.assign(new Error("connect failed"), {
|
||||
code: "ECONNREFUSED",
|
||||
syscall: "connect",
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
dispatcher.sendFinalReply(fixture.payload);
|
||||
dispatcher.markComplete();
|
||||
await dispatcher.waitForIdle();
|
||||
|
||||
expect(
|
||||
(
|
||||
loadSessionEntry({
|
||||
sessionKey: fixture.sessionKey,
|
||||
storePath: fixture.storePath,
|
||||
}) as InternalSessionEntry
|
||||
)?.pendingFinalDelivery?.deliveries,
|
||||
).toEqual([{ id: "delivery-1", state: "prepared" }]);
|
||||
} finally {
|
||||
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("suppresses a second direct call after the exact delivery is terminal", async () => {
|
||||
const fixture = await makePendingFinalFixture();
|
||||
const deliver = vi.fn(async () => {});
|
||||
try {
|
||||
const first = createReplyDispatcher({ deliver });
|
||||
first.sendFinalReply(fixture.payload);
|
||||
first.markComplete();
|
||||
await first.waitForIdle();
|
||||
|
||||
const second = createReplyDispatcher({ deliver });
|
||||
second.sendFinalReply(fixture.payload);
|
||||
second.markComplete();
|
||||
await second.waitForIdle();
|
||||
|
||||
expect(deliver).toHaveBeenCalledOnce();
|
||||
expect(second.getCancelledCounts?.().final).toBe(1);
|
||||
} finally {
|
||||
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("suppresses a direct call whose persisted owner was replaced", async () => {
|
||||
const fixture = await makePendingFinalFixture();
|
||||
const current = loadSessionEntry({
|
||||
sessionKey: fixture.sessionKey,
|
||||
storePath: fixture.storePath,
|
||||
}) as InternalSessionEntry;
|
||||
await replaceSessionEntry(
|
||||
{ sessionKey: fixture.sessionKey, storePath: fixture.storePath },
|
||||
{
|
||||
...current,
|
||||
pendingFinalDelivery: {
|
||||
...current.pendingFinalDelivery!,
|
||||
intentId: "replacement-intent",
|
||||
},
|
||||
},
|
||||
);
|
||||
const deliver = vi.fn(async () => {});
|
||||
try {
|
||||
const dispatcher = createReplyDispatcher({ deliver });
|
||||
dispatcher.sendFinalReply(fixture.payload);
|
||||
dispatcher.markComplete();
|
||||
await dispatcher.waitForIdle();
|
||||
|
||||
expect(deliver).not.toHaveBeenCalled();
|
||||
expect(dispatcher.getCancelledCounts?.().final).toBe(1);
|
||||
} finally {
|
||||
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("records policy suppression before awaiting cancellation observers", async () => {
|
||||
const fixture = await makePendingFinalFixture();
|
||||
const observerStarted = createDeferred();
|
||||
const releaseObserver = createDeferred();
|
||||
try {
|
||||
const dispatcher = createReplyDispatcher({
|
||||
beforeDeliver: () => null,
|
||||
deliver: async () => {},
|
||||
onBeforeDeliverCancelled: async () => {
|
||||
observerStarted.resolve();
|
||||
await releaseObserver.promise;
|
||||
},
|
||||
});
|
||||
dispatcher.sendFinalReply(fixture.payload);
|
||||
dispatcher.markComplete();
|
||||
await observerStarted.promise;
|
||||
|
||||
expect(
|
||||
(
|
||||
loadSessionEntry({
|
||||
sessionKey: fixture.sessionKey,
|
||||
storePath: fixture.storePath,
|
||||
}) as InternalSessionEntry
|
||||
)?.pendingFinalDelivery?.deliveries,
|
||||
).toEqual([{ id: "delivery-1", state: "suppressed" }]);
|
||||
|
||||
releaseObserver.resolve();
|
||||
await dispatcher.waitForIdle();
|
||||
} finally {
|
||||
releaseObserver.resolve();
|
||||
await fs.rm(fixture.tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -21,8 +21,7 @@ import {
|
||||
} from "./dispatch-from-config.payloads.js";
|
||||
import {
|
||||
clearPendingFinalDeliveryAfterSuccess,
|
||||
capturePendingFinalDeliveryIdentity,
|
||||
reconcilePendingFinalDeliveryAfterSettlement,
|
||||
suppressPendingFinalDelivery,
|
||||
} from "./dispatch-from-config.pending-final.js";
|
||||
import type { ReplyDispatchDeliveryOutcome } from "./reply-dispatcher.js";
|
||||
|
||||
@@ -48,26 +47,15 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
|
||||
sendPolicyDenied,
|
||||
sessionAgentId,
|
||||
sessionKey,
|
||||
sessionStoreEntry,
|
||||
suppressDelivery,
|
||||
throwIfDispatchOperationAborted,
|
||||
turnLedger,
|
||||
waitForPendingDirectBlockReplyDelivery,
|
||||
} = state;
|
||||
const replies = replyResult ? (Array.isArray(replyResult) ? replyResult : [replyResult]) : [];
|
||||
const pendingFinalDelivery = {
|
||||
storePath: sessionStoreEntry.storePath,
|
||||
sessionKey: sessionStoreEntry.sessionKey ?? sessionKey,
|
||||
};
|
||||
const replyPendingIntentIds = new Set(
|
||||
replies
|
||||
.map((reply) => getReplyPayloadMetadata(reply)?.pendingFinalDeliveryIntentId)
|
||||
.filter((intentId): intentId is string => Boolean(intentId)),
|
||||
);
|
||||
const pendingFinalDeliveryIdentity = capturePendingFinalDeliveryIdentity({
|
||||
...pendingFinalDelivery,
|
||||
intentId: replyPendingIntentIds.size === 1 ? [...replyPendingIntentIds][0] : undefined,
|
||||
});
|
||||
const pendingFinalDeliveryIdentity = replies
|
||||
.map((reply) => getReplyPayloadMetadata(reply)?.pendingFinalDeliveryCompletion)
|
||||
.find((completion) => completion !== undefined);
|
||||
// Final delivery is outside the progress wrappers. Wait until every source-ordered callback
|
||||
// has at least started so a delayed tool/reasoning transition cannot appear after the final.
|
||||
if (state.preserveProgressCallbackStartOrder) {
|
||||
@@ -84,10 +72,7 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
|
||||
let routedFinalCount = 0;
|
||||
let attemptedFinalDelivery = false;
|
||||
let finalDeliveryFailed = false;
|
||||
const finalDeliveries: Array<{
|
||||
outcome: Promise<ReplyDispatchDeliveryOutcome>;
|
||||
payload: ReplyPayload;
|
||||
}> = [];
|
||||
const finalDeliveries: Promise<ReplyDispatchDeliveryOutcome>[] = [];
|
||||
let allQueuedFinalsObserved = true;
|
||||
const sentFinalPayloadDedupeKeys = new Set<string>();
|
||||
let deferredTtsTextPending = state.progressState.accumulatedBlockTtsText;
|
||||
@@ -96,9 +81,11 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
|
||||
// Durable reasoning is a channel-owned lane; generic channels keep the
|
||||
// historical suppression unless they explicitly opt in.
|
||||
if (reply.isReasoning === true && !state.reasoningPayloadsEnabled) {
|
||||
await suppressPendingFinalDelivery(reply);
|
||||
continue;
|
||||
}
|
||||
if (reply.isCommentary === true && !state.commentaryPayloadsEnabled) {
|
||||
await suppressPendingFinalDelivery(reply);
|
||||
continue;
|
||||
}
|
||||
if (suppressDelivery && !shouldDeliverDespiteSourceReplySuppression(reply, state)) {
|
||||
@@ -116,10 +103,12 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
|
||||
].join(" "),
|
||||
);
|
||||
}
|
||||
await suppressPendingFinalDelivery(reply);
|
||||
continue;
|
||||
}
|
||||
const finalPayloadDedupeKey = createFinalDispatchPayloadDedupeKey(reply);
|
||||
if (sentFinalPayloadDedupeKeys.has(finalPayloadDedupeKey)) {
|
||||
await suppressPendingFinalDelivery(reply);
|
||||
continue;
|
||||
}
|
||||
sentFinalPayloadDedupeKeys.add(finalPayloadDedupeKey);
|
||||
@@ -141,6 +130,7 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
|
||||
}
|
||||
if (finalReply.dedupedAgainstBlock) {
|
||||
// The delivering block already settled into the turn ledger.
|
||||
await suppressPendingFinalDelivery(reply);
|
||||
continue;
|
||||
}
|
||||
attemptedFinalDelivery = true;
|
||||
@@ -148,7 +138,7 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
|
||||
routedFinalCount += finalReply.routedFinalCount;
|
||||
if (finalReply.queuedFinal) {
|
||||
if (finalReply.dispatcherOutcome) {
|
||||
finalDeliveries.push({ outcome: finalReply.dispatcherOutcome, payload: reply });
|
||||
finalDeliveries.push(finalReply.dispatcherOutcome);
|
||||
} else {
|
||||
allQueuedFinalsObserved = false;
|
||||
}
|
||||
@@ -162,19 +152,9 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
|
||||
if (queuedFinal && allQueuedFinalsObserved) {
|
||||
// Delivery observers run from the queue itself, so direct low-level callers
|
||||
// reconcile too; the settle task only makes lifecycle owners await it.
|
||||
const reconcilePendingFinal = Promise.all(
|
||||
finalDeliveries.map(async (delivery) => ({
|
||||
outcome: await delivery.outcome,
|
||||
payload: delivery.payload,
|
||||
})),
|
||||
)
|
||||
.then(async (deliveries) => {
|
||||
await reconcilePendingFinalDeliveryAfterSettlement({
|
||||
...pendingFinalDelivery,
|
||||
deliveries,
|
||||
identity: pendingFinalDeliveryIdentity,
|
||||
replies,
|
||||
});
|
||||
const reconcilePendingFinal = Promise.all(finalDeliveries)
|
||||
.then(async () => {
|
||||
await clearPendingFinalDeliveryAfterSuccess(pendingFinalDeliveryIdentity);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
logVerbose(
|
||||
@@ -185,10 +165,7 @@ export async function finalizeDispatchAndAudit(state: ExecuteDispatchReadyState)
|
||||
} else {
|
||||
// Routed delivery has a transport result already. Custom dispatchers that
|
||||
// do not expose the core observer retain the legacy queue-admission behavior.
|
||||
await clearPendingFinalDeliveryAfterSuccess({
|
||||
...pendingFinalDelivery,
|
||||
identity: pendingFinalDeliveryIdentity,
|
||||
});
|
||||
await clearPendingFinalDeliveryAfterSuccess(pendingFinalDeliveryIdentity);
|
||||
}
|
||||
// Register successful queued cleanup before honoring a late abort. The
|
||||
// outer settle owner still runs it from finally (#89115).
|
||||
|
||||
@@ -4,11 +4,14 @@ import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
|
||||
import type { ReplyPayload } from "../reply-payload.js";
|
||||
import {
|
||||
capturePendingFinalDeliveryIdentity,
|
||||
getReplyPayloadMetadata,
|
||||
setReplyPayloadMetadata,
|
||||
type ReplyPayload,
|
||||
} from "../reply-payload.js";
|
||||
import {
|
||||
clearPendingFinalDeliveryAfterSuccess,
|
||||
reconcilePendingFinalDeliveryAfterSettlement,
|
||||
suppressPendingFinalDelivery,
|
||||
} from "./dispatch-from-config.pending-final.js";
|
||||
import { retireTerminalRestartRecoverySourceClaim } from "./restart-recovery-claim.js";
|
||||
|
||||
@@ -28,6 +31,7 @@ describe("pending final delivery restart proof", () => {
|
||||
|
||||
async function writePendingFinal(
|
||||
beforeAgentReplyState: "continue" | "handled-reply",
|
||||
state: "prepared" | "delivered" = "delivered",
|
||||
): Promise<void> {
|
||||
const entry: SessionEntry = {
|
||||
sessionId: "session",
|
||||
@@ -40,6 +44,7 @@ describe("pending final delivery restart proof", () => {
|
||||
text: "hook reply",
|
||||
createdAt: 1,
|
||||
intentId: "intent-1",
|
||||
deliveries: [{ id: "delivery-1", state }],
|
||||
},
|
||||
restartRecoveryBeforeAgentReplyState: beforeAgentReplyState,
|
||||
restartRecoveryForceSafeTools: beforeAgentReplyState === "handled-reply" ? true : undefined,
|
||||
@@ -48,17 +53,28 @@ describe("pending final delivery restart proof", () => {
|
||||
await replaceSessionEntry({ storePath, sessionKey }, entry);
|
||||
}
|
||||
|
||||
function pendingFinalPayload(deliveryId = "delivery-1"): ReplyPayload {
|
||||
const payload: ReplyPayload = { text: "hook reply" };
|
||||
setReplyPayloadMetadata(payload, {
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId,
|
||||
intentId: "intent-1",
|
||||
sessionId: "session",
|
||||
sessionKey,
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
return payload;
|
||||
}
|
||||
|
||||
it.each(["continue", "handled-reply"] as const)(
|
||||
"clears %s provenance only after the exact pending intent succeeds",
|
||||
async (beforeAgentReplyState) => {
|
||||
await writePendingFinal(beforeAgentReplyState);
|
||||
const identity = capturePendingFinalDeliveryIdentity({
|
||||
intentId: "intent-1",
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
const identity =
|
||||
getReplyPayloadMetadata(pendingFinalPayload())?.pendingFinalDeliveryCompletion;
|
||||
|
||||
await clearPendingFinalDeliveryAfterSuccess({ identity, sessionKey, storePath });
|
||||
await clearPendingFinalDeliveryAfterSuccess(identity);
|
||||
|
||||
const entry = loadSessionEntry({ sessionKey, storePath }) as SessionEntry | undefined;
|
||||
expect(entry?.pendingFinalDelivery).toBeUndefined();
|
||||
@@ -87,18 +103,25 @@ describe("pending final delivery restart proof", () => {
|
||||
kind: "transport-only",
|
||||
createdAt: Date.now(),
|
||||
intentId: "intent-media",
|
||||
deliveries: [{ id: "delivery-media", state: "delivered" }],
|
||||
},
|
||||
restartRecoveryBeforeAgentReplyState: "handled-unrecoverable",
|
||||
restartRecoverySourceIngress: "channel",
|
||||
};
|
||||
await replaceSessionEntry({ storePath, sessionKey }, entry);
|
||||
const identity = capturePendingFinalDeliveryIdentity({
|
||||
intentId: "intent-media",
|
||||
sessionKey,
|
||||
storePath,
|
||||
const payload: ReplyPayload = { mediaUrl: "https://example.test/image.png" };
|
||||
setReplyPayloadMetadata(payload, {
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId: "delivery-media",
|
||||
intentId: "intent-media",
|
||||
sessionId: "session",
|
||||
sessionKey,
|
||||
storePath,
|
||||
},
|
||||
});
|
||||
const identity = getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion;
|
||||
|
||||
await clearPendingFinalDeliveryAfterSuccess({ identity, sessionKey, storePath });
|
||||
await clearPendingFinalDeliveryAfterSuccess(identity);
|
||||
|
||||
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
|
||||
status: "done",
|
||||
@@ -109,32 +132,40 @@ describe("pending final delivery restart proof", () => {
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps normal-turn provenance when transport fails before delivery", async () => {
|
||||
await writePendingFinal("continue");
|
||||
const identity = capturePendingFinalDeliveryIdentity({
|
||||
intentId: "intent-1",
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
const payload: ReplyPayload = { text: "hook reply" };
|
||||
|
||||
await reconcilePendingFinalDeliveryAfterSettlement({
|
||||
deliveries: [{ outcome: "failed-before-deliver", payload }],
|
||||
identity,
|
||||
replies: [payload],
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
|
||||
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: "hook reply",
|
||||
intentId: "intent-1",
|
||||
it("clears a skipped turn only after every sendable final is suppressed", async () => {
|
||||
await writePendingFinal("continue", "prepared");
|
||||
await replaceSessionEntry(
|
||||
{ storePath, sessionKey },
|
||||
{
|
||||
...(loadSessionEntry({ sessionKey, storePath }) as SessionEntry),
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: "hook reply",
|
||||
createdAt: 1,
|
||||
intentId: "intent-1",
|
||||
deliveries: [
|
||||
{ id: "delivery-1", state: "prepared" },
|
||||
{ id: "delivery-2", state: "prepared" },
|
||||
],
|
||||
},
|
||||
},
|
||||
restartRecoveryBeforeAgentReplyState: "continue",
|
||||
restartRecoverySourceIngress: "channel",
|
||||
});
|
||||
);
|
||||
|
||||
await suppressPendingFinalDelivery(pendingFinalPayload("delivery-1"));
|
||||
|
||||
expect(
|
||||
(loadSessionEntry({ sessionKey, storePath }) as SessionEntry).pendingFinalDelivery
|
||||
?.deliveries,
|
||||
).toEqual([
|
||||
{ id: "delivery-1", state: "suppressed" },
|
||||
{ id: "delivery-2", state: "prepared" },
|
||||
]);
|
||||
|
||||
await suppressPendingFinalDelivery(pendingFinalPayload("delivery-2"));
|
||||
|
||||
expect(
|
||||
(loadSessionEntry({ sessionKey, storePath }) as SessionEntry).pendingFinalDelivery,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not retire a source while its terminal provider outcome is unknown", async () => {
|
||||
|
||||
@@ -1,245 +1,73 @@
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { buildRestartRecoveryClaimCleanupPatch } from "../../config/sessions/restart-recovery-state.js";
|
||||
import { updateSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import { settlePendingFinalDelivery } from "../../infra/outbound/delivery-completion.js";
|
||||
import {
|
||||
loadSessionEntryReadOnly,
|
||||
updateSessionEntry,
|
||||
} from "../../config/sessions/session-accessor.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
|
||||
import type { ReplyPayload } from "../reply-payload.js";
|
||||
import { getReplyPayloadMetadata } from "../reply-payload.js";
|
||||
import {
|
||||
buildPendingFinalDeliveryText,
|
||||
sanitizePendingFinalDeliveryText,
|
||||
} from "./pending-final-delivery.js";
|
||||
import type { ReplyDispatchDeliveryOutcome } from "./reply-dispatcher.js";
|
||||
getReplyPayloadMetadata,
|
||||
type ReplyPayload,
|
||||
type ReplyPayloadMetadata,
|
||||
} from "../reply-payload.js";
|
||||
|
||||
type SettledFinalDelivery = {
|
||||
outcome: ReplyDispatchDeliveryOutcome;
|
||||
payload: ReplyPayload;
|
||||
};
|
||||
type PendingFinalDeliveryIdentity = NonNullable<
|
||||
ReplyPayloadMetadata["pendingFinalDeliveryCompletion"]
|
||||
>;
|
||||
|
||||
type PendingFinalDeliveryIdentity = {
|
||||
createdAt?: number;
|
||||
intentId?: string;
|
||||
present: boolean;
|
||||
text?: string;
|
||||
};
|
||||
|
||||
function buildPendingFinalDeliveryCleanupPatch(entry: SessionEntry): Partial<SessionEntry> {
|
||||
// An active receipt/claim may outlive outer reply settlement. Only claimless pending finals
|
||||
// borrow hook provenance until their exact transport intent settles.
|
||||
const clearsRestartRecoveryProof =
|
||||
normalizeOptionalString(entry.restartRecoveryDeliveryRunId) === undefined;
|
||||
const completesHookHandledTurn =
|
||||
clearsRestartRecoveryProof &&
|
||||
(entry.restartRecoveryBeforeAgentReplyState === "handled-reply" ||
|
||||
entry.restartRecoveryBeforeAgentReplyState === "handled-unrecoverable");
|
||||
const endedAt = completesHookHandledTurn ? Date.now() : undefined;
|
||||
return {
|
||||
pendingFinalDelivery: undefined,
|
||||
...(clearsRestartRecoveryProof
|
||||
? {
|
||||
restartRecoveryBeforeAgentReplyState: undefined,
|
||||
restartRecoverySourceIngress: undefined,
|
||||
restartRecoveryForceSafeTools: undefined,
|
||||
}
|
||||
: {}),
|
||||
...(endedAt !== undefined
|
||||
? {
|
||||
abortedLastRun: false,
|
||||
endedAt,
|
||||
lifecycleRunId: undefined,
|
||||
runtimeMs:
|
||||
typeof entry.startedAt === "number"
|
||||
? Math.max(0, endedAt - entry.startedAt)
|
||||
: undefined,
|
||||
status: "done" as const,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
export async function suppressPendingFinalDelivery(payload: ReplyPayload): Promise<void> {
|
||||
const completion = getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion;
|
||||
if (completion) {
|
||||
await settlePendingFinalDelivery({ kind: "pending-final", ...completion }, "suppressed", [
|
||||
"prepared",
|
||||
]);
|
||||
await clearPendingFinalDeliveryAfterSuccess(completion);
|
||||
}
|
||||
}
|
||||
|
||||
function matchesPendingFinalDeliveryIdentity(
|
||||
entry: SessionEntry,
|
||||
expected: PendingFinalDeliveryIdentity,
|
||||
): boolean {
|
||||
const pending = entry.pendingFinalDelivery;
|
||||
const currentPresent = pending !== undefined;
|
||||
if (currentPresent !== expected.present) {
|
||||
return false;
|
||||
}
|
||||
if (expected.intentId) {
|
||||
return pending?.intentId === expected.intentId;
|
||||
}
|
||||
return (
|
||||
pending?.createdAt === expected.createdAt &&
|
||||
(pending?.kind === "replayable" ? pending.text : undefined) === expected.text
|
||||
);
|
||||
}
|
||||
|
||||
export async function clearPendingFinalDeliveryAfterSuccess(params: {
|
||||
identity?: PendingFinalDeliveryIdentity;
|
||||
storePath?: string;
|
||||
sessionKey?: string;
|
||||
}): Promise<void> {
|
||||
const identity = params.identity;
|
||||
if (!params.storePath || !params.sessionKey || !identity?.present) {
|
||||
export async function clearPendingFinalDeliveryAfterSuccess(
|
||||
identity?: PendingFinalDeliveryIdentity,
|
||||
): Promise<void> {
|
||||
if (!identity) {
|
||||
return;
|
||||
}
|
||||
await updateSessionEntry(
|
||||
{ storePath: params.storePath, sessionKey: params.sessionKey },
|
||||
async (entry) => {
|
||||
if (!matchesPendingFinalDeliveryIdentity(entry, identity)) {
|
||||
return null;
|
||||
}
|
||||
if (!entry.pendingFinalDelivery) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
...buildPendingFinalDeliveryCleanupPatch(entry),
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
{ skipMaintenance: true, takeCacheOwnership: true },
|
||||
);
|
||||
}
|
||||
|
||||
export function capturePendingFinalDeliveryIdentity(params: {
|
||||
intentId?: string;
|
||||
storePath?: string;
|
||||
sessionKey?: string;
|
||||
}): PendingFinalDeliveryIdentity | undefined {
|
||||
if (!params.storePath || !params.sessionKey) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const entry = loadSessionEntryReadOnly({
|
||||
storePath: params.storePath,
|
||||
sessionKey: params.sessionKey,
|
||||
hydrateSkillPromptRefs: false,
|
||||
readConsistency: "latest",
|
||||
});
|
||||
const pending = entry?.pendingFinalDelivery;
|
||||
if (params.intentId && pending?.intentId !== params.intentId) {
|
||||
return { present: false };
|
||||
}
|
||||
return {
|
||||
present: pending !== undefined,
|
||||
intentId: params.intentId ?? pending?.intentId,
|
||||
createdAt: pending?.createdAt,
|
||||
text: pending?.kind === "replayable" ? pending.text : undefined,
|
||||
};
|
||||
} catch {
|
||||
return params.intentId ? { present: true, intentId: params.intentId } : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function buildPendingFinalDeliveryRetryText(payloads: ReplyPayload[]): string {
|
||||
return sanitizePendingFinalDeliveryText(
|
||||
payloads
|
||||
.map(
|
||||
(payload) =>
|
||||
getReplyPayloadMetadata(payload)?.pendingFinalDeliveryRetryText ??
|
||||
buildPendingFinalDeliveryText([payload]),
|
||||
)
|
||||
.filter(Boolean)
|
||||
.join("\n\n"),
|
||||
);
|
||||
}
|
||||
|
||||
function resolvePendingFinalDeliveryPayloads(params: {
|
||||
intentId?: string;
|
||||
pendingText: string;
|
||||
replies: ReplyPayload[];
|
||||
}): ReplyPayload[] | undefined {
|
||||
const intentReplies = params.intentId
|
||||
? params.replies.filter((reply) => {
|
||||
const metadata = getReplyPayloadMetadata(reply);
|
||||
return (
|
||||
metadata?.pendingFinalDeliveryIntentId === params.intentId &&
|
||||
metadata?.pendingFinalDeliveryRetryText !== undefined
|
||||
);
|
||||
})
|
||||
: [];
|
||||
const intentContributors = intentReplies.filter(
|
||||
(reply) => getReplyPayloadMetadata(reply)?.pendingFinalDeliveryRetryText,
|
||||
);
|
||||
const intentText = buildPendingFinalDeliveryRetryText(intentContributors);
|
||||
if (
|
||||
intentReplies.length > 0 &&
|
||||
intentText.replace(/\s+/g, " ").trim() === params.pendingText.replace(/\s+/g, " ").trim()
|
||||
) {
|
||||
return intentContributors;
|
||||
}
|
||||
const contributingReplies = params.replies.filter(
|
||||
(reply) => buildPendingFinalDeliveryText([reply]) !== "",
|
||||
);
|
||||
if (buildPendingFinalDeliveryText(contributingReplies) === params.pendingText) {
|
||||
return contributingReplies;
|
||||
}
|
||||
const exactMatches = contributingReplies.filter(
|
||||
(reply) => buildPendingFinalDeliveryText([reply]) === params.pendingText,
|
||||
);
|
||||
return exactMatches.length === 1 ? exactMatches : undefined;
|
||||
}
|
||||
|
||||
export async function reconcilePendingFinalDeliveryAfterSettlement(params: {
|
||||
deliveries: SettledFinalDelivery[];
|
||||
identity?: PendingFinalDeliveryIdentity;
|
||||
replies: ReplyPayload[];
|
||||
storePath?: string;
|
||||
sessionKey?: string;
|
||||
}): Promise<void> {
|
||||
const identity = params.identity;
|
||||
if (!params.storePath || !params.sessionKey || !identity?.present) {
|
||||
return;
|
||||
}
|
||||
await updateSessionEntry(
|
||||
{ storePath: params.storePath, sessionKey: params.sessionKey },
|
||||
async (entry) => {
|
||||
if (!matchesPendingFinalDeliveryIdentity(entry, identity)) {
|
||||
return null;
|
||||
}
|
||||
const pending = entry.pendingFinalDelivery;
|
||||
if (!pending) {
|
||||
return null;
|
||||
}
|
||||
const pendingPayloads =
|
||||
pending.kind === "replayable"
|
||||
? resolvePendingFinalDeliveryPayloads({
|
||||
intentId: identity.intentId,
|
||||
pendingText: pending.text,
|
||||
replies: params.replies,
|
||||
})
|
||||
: undefined;
|
||||
const pendingPayloadSet = pendingPayloads ? new Set(pendingPayloads) : undefined;
|
||||
const relevantDeliveries = pendingPayloadSet
|
||||
? params.deliveries.filter((delivery) => pendingPayloadSet.has(delivery.payload))
|
||||
: params.deliveries;
|
||||
const ownsEveryPendingPayload =
|
||||
!pendingPayloadSet || relevantDeliveries.length === pendingPayloadSet.size;
|
||||
const failedBeforeDeliver = relevantDeliveries.filter(
|
||||
(delivery) => delivery.outcome === "failed-before-deliver",
|
||||
);
|
||||
|
||||
{ storePath: identity.storePath, sessionKey: identity.sessionKey },
|
||||
(entry) => {
|
||||
const recoveryRunId = normalizeOptionalString(entry.restartRecoveryDeliveryRunId);
|
||||
const deliveries = entry.pendingFinalDelivery?.deliveries;
|
||||
if (
|
||||
relevantDeliveries.length > 0 &&
|
||||
failedBeforeDeliver.length === relevantDeliveries.length
|
||||
entry.sessionId !== identity.sessionId ||
|
||||
entry.pendingFinalDelivery?.intentId !== identity.intentId ||
|
||||
!deliveries?.length ||
|
||||
!deliveries.every(({ state }) => state === "delivered" || state === "suppressed") ||
|
||||
(recoveryRunId !== undefined && recoveryRunId !== identity.recoveryRunId)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (pendingPayloadSet && ownsEveryPendingPayload && failedBeforeDeliver.length > 0) {
|
||||
const retryText = buildPendingFinalDeliveryRetryText(
|
||||
failedBeforeDeliver.map((delivery) => delivery.payload),
|
||||
);
|
||||
if (retryText && pending.kind === "replayable") {
|
||||
return {
|
||||
pendingFinalDelivery: { ...pending, text: retryText },
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
}
|
||||
}
|
||||
const completesHookTurn =
|
||||
recoveryRunId === undefined &&
|
||||
(entry.restartRecoveryBeforeAgentReplyState === "handled-reply" ||
|
||||
entry.restartRecoveryBeforeAgentReplyState === "handled-unrecoverable");
|
||||
const endedAt = completesHookTurn ? Date.now() : undefined;
|
||||
return {
|
||||
...buildPendingFinalDeliveryCleanupPatch(entry),
|
||||
...(recoveryRunId
|
||||
? buildRestartRecoveryClaimCleanupPatch({ entry, recordTerminalSource: true })
|
||||
: {
|
||||
restartRecoveryBeforeAgentReplyState: undefined,
|
||||
restartRecoverySourceIngress: undefined,
|
||||
restartRecoveryForceSafeTools: undefined,
|
||||
}),
|
||||
pendingFinalDelivery: undefined,
|
||||
...(endedAt === undefined
|
||||
? {}
|
||||
: {
|
||||
abortedLastRun: false,
|
||||
endedAt,
|
||||
lifecycleRunId: undefined,
|
||||
runtimeMs:
|
||||
typeof entry.startedAt === "number"
|
||||
? Math.max(0, endedAt - entry.startedAt)
|
||||
: undefined,
|
||||
status: "done" as const,
|
||||
}),
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createDeferred } from "../../../test/helpers/promise.js";
|
||||
import { clearAgentHarnesses } from "../../agents/harness/registry.js";
|
||||
import {
|
||||
OutboundDeliveryError,
|
||||
PlatformMessageNotDispatchedError,
|
||||
} from "../../infra/outbound/deliver-types.js";
|
||||
import type { PluginHookReplyDispatchResult } from "../../plugins/hooks.test-fixtures.js";
|
||||
import { getPluginRuntimeGatewayRequestScope } from "../../plugins/runtime/gateway-request-scope.js";
|
||||
import { createInternalHookEventPayload } from "../../test-utils/internal-hook-event-payload.js";
|
||||
@@ -55,9 +59,42 @@ function firstReplyDispatchCall() {
|
||||
|
||||
function pendingFinalDelivery(
|
||||
text: string,
|
||||
overrides: { createdAt?: number; context?: Record<string, unknown>; intentId?: string } = {},
|
||||
overrides: {
|
||||
createdAt?: number;
|
||||
context?: Record<string, unknown>;
|
||||
deliveries?: Array<{
|
||||
id: string;
|
||||
state: "prepared" | "queued" | "delivered" | "suppressed" | "unknown";
|
||||
}>;
|
||||
intentId?: string;
|
||||
} = {},
|
||||
) {
|
||||
return { kind: "replayable" as const, text, createdAt: 1, ...overrides };
|
||||
return {
|
||||
kind: "replayable" as const,
|
||||
text,
|
||||
createdAt: 1,
|
||||
intentId: "intent-1",
|
||||
deliveries: [{ id: "delivery-1", state: "prepared" as const }],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function pendingFinalReply(
|
||||
text: string,
|
||||
overrides: { deliveryId?: string; intentId?: string } = {},
|
||||
): ReplyPayload {
|
||||
return setReplyPayloadMetadata(
|
||||
{ text },
|
||||
{
|
||||
pendingFinalDeliveryCompletion: {
|
||||
deliveryId: overrides.deliveryId ?? "delivery-1",
|
||||
intentId: overrides.intentId ?? "intent-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
storePath: "/tmp/mock-sessions.json",
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
@@ -217,6 +254,93 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("clears pending final delivery after final dispatch succeeds", async () => {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pendingFinalDelivery("durable reply", {
|
||||
context: { source: "heartbeat" },
|
||||
}),
|
||||
};
|
||||
sessionStoreMocks.loadSessionStore.mockClear();
|
||||
mocks.routeReply.mockResolvedValue({ ok: true, delivered: true, messageId: "mock" });
|
||||
|
||||
const deliver = vi.fn().mockResolvedValue(undefined);
|
||||
const dispatcher = createReplyDispatcher({ deliver });
|
||||
const result = await dispatchReplyFromConfig({
|
||||
ctx: createHookCtx(),
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyResolver: async () => pendingFinalReply("durable reply"),
|
||||
});
|
||||
await dispatcher.waitForIdle();
|
||||
await vi.waitFor(() => {
|
||||
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined();
|
||||
});
|
||||
|
||||
expect(result.queuedFinal).toBe(true);
|
||||
expect(sessionStoreMocks.loadSessionStoreEntry).toHaveBeenCalledWith({
|
||||
agentId: "test",
|
||||
storePath: "/tmp/mock-sessions.json",
|
||||
sessionKey: "agent:test:session",
|
||||
readConsistency: "latest",
|
||||
clone: false,
|
||||
});
|
||||
expect(sessionStoreMocks.loadSessionStore).not.toHaveBeenCalled();
|
||||
expect(deliver).toHaveBeenCalledOnce();
|
||||
expect(sessionStoreMocks.updateSessionEntry).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("clears pending final delivery when abort fires after a successful final send (#89115)", async () => {
|
||||
// Regression for #89115: an abort that lands after the final reply has
|
||||
// shipped (here, during sendFinalReply) must still clear the pending-final
|
||||
// bookkeeping — otherwise pendingFinalDelivery stays true and the get-reply
|
||||
// redelivery short-circuit silently blocks every later inbound.
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pendingFinalDelivery("durable reply", {
|
||||
context: { source: "heartbeat" },
|
||||
intentId: "intent-89115",
|
||||
}),
|
||||
};
|
||||
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
|
||||
existing: sessionStoreMocks.currentEntry,
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
const deliver = vi.fn().mockResolvedValue(undefined);
|
||||
const dispatcher = createReplyDispatcher({ deliver });
|
||||
const sendFinalReply = dispatcher.sendFinalReply.bind(dispatcher);
|
||||
vi.spyOn(dispatcher, "sendFinalReply").mockImplementation((payload) => {
|
||||
const queued = sendFinalReply(payload);
|
||||
abortController.abort();
|
||||
return queued;
|
||||
});
|
||||
|
||||
const result = await withReplyDispatcher({
|
||||
dispatcher,
|
||||
run: () =>
|
||||
dispatchReplyFromConfig({
|
||||
ctx: createHookCtx(),
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyOptions: { abortSignal: abortController.signal },
|
||||
replyResolver: async () =>
|
||||
pendingFinalReply("durable reply", { intentId: "intent-89115" }),
|
||||
}),
|
||||
});
|
||||
|
||||
// Abort landed after delivery: the run is still surfaced as aborted
|
||||
// (queuedFinal:false), but the pending-final state is fully cleared.
|
||||
expect(dispatcher.sendFinalReply).toHaveBeenCalledOnce();
|
||||
expect(deliver).toHaveBeenCalledOnce();
|
||||
expect(result.queuedFinal).toBe(false);
|
||||
expect(sessionStoreMocks.updateSessionEntry).toHaveBeenCalledTimes(3);
|
||||
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves pending final delivery when final dispatch fails", async () => {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
@@ -248,6 +372,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
try {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pendingFinalDelivery("durable reply", {
|
||||
context: { channel: "whatsapp", to: "+1000" },
|
||||
@@ -273,7 +398,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
ctx: createHookCtx(),
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyResolver: async () => ({ text: "durable reply" }),
|
||||
replyResolver: async () => pendingFinalReply("durable reply"),
|
||||
}),
|
||||
});
|
||||
await hookStarted.promise;
|
||||
@@ -299,11 +424,66 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("clears pending final delivery when a later queued final succeeds", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pendingFinalDelivery("durable reply"),
|
||||
};
|
||||
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
|
||||
existing: sessionStoreMocks.currentEntry,
|
||||
});
|
||||
const hookStarted = createDeferred();
|
||||
const deliver = vi.fn().mockResolvedValue(undefined);
|
||||
let hookCalls = 0;
|
||||
const dispatcher = createReplyDispatcher({
|
||||
deliver,
|
||||
beforeDeliver: (payload) => {
|
||||
hookCalls += 1;
|
||||
if (hookCalls === 1) {
|
||||
hookStarted.resolve();
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
return payload;
|
||||
},
|
||||
});
|
||||
|
||||
const resultPromise = withReplyDispatcher({
|
||||
dispatcher,
|
||||
run: () =>
|
||||
dispatchReplyFromConfig({
|
||||
ctx: createHookCtx(),
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyResolver: async () => [{ text: "first" }, pendingFinalReply("durable reply")],
|
||||
}),
|
||||
});
|
||||
await hookStarted.promise;
|
||||
await vi.advanceTimersByTimeAsync(15_000);
|
||||
await resultPromise;
|
||||
|
||||
expect(deliver).toHaveBeenCalledOnce();
|
||||
expect(deliver).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ text: "durable reply" }),
|
||||
expect.objectContaining({ kind: "final" }),
|
||||
);
|
||||
expect(dispatcher.getFailedCounts?.()).toEqual({ tool: 0, block: 0, final: 1 });
|
||||
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves the durable final when an earlier auxiliary final succeeds", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pendingFinalDelivery("durable reply"),
|
||||
};
|
||||
@@ -332,7 +512,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
ctx: createHookCtx(),
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyResolver: async () => [{ text: "auxiliary" }, { text: "durable reply" }],
|
||||
replyResolver: async () => [{ text: "auxiliary" }, pendingFinalReply("durable reply")],
|
||||
}),
|
||||
});
|
||||
await hookStarted.promise;
|
||||
@@ -353,11 +533,74 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("records each pending-final delivery without rewriting aggregate text", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pendingFinalDelivery("auxiliary\n\ndurable reply", {
|
||||
deliveries: [
|
||||
{ id: "delivery-auxiliary", state: "prepared" },
|
||||
{ id: "delivery-durable", state: "prepared" },
|
||||
],
|
||||
}),
|
||||
};
|
||||
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
|
||||
existing: sessionStoreMocks.currentEntry,
|
||||
});
|
||||
const hookStarted = createDeferred();
|
||||
let hookCalls = 0;
|
||||
const dispatcher = createReplyDispatcher({
|
||||
deliver: vi.fn().mockResolvedValue(undefined),
|
||||
beforeDeliver: (payload) => {
|
||||
hookCalls += 1;
|
||||
if (hookCalls === 2) {
|
||||
hookStarted.resolve();
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
return payload;
|
||||
},
|
||||
});
|
||||
|
||||
const resultPromise = withReplyDispatcher({
|
||||
dispatcher,
|
||||
run: () =>
|
||||
dispatchReplyFromConfig({
|
||||
ctx: createHookCtx(),
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyResolver: async () => [
|
||||
pendingFinalReply("auxiliary", { deliveryId: "delivery-auxiliary" }),
|
||||
pendingFinalReply("durable reply", { deliveryId: "delivery-durable" }),
|
||||
],
|
||||
}),
|
||||
});
|
||||
await hookStarted.promise;
|
||||
await vi.advanceTimersByTimeAsync(15_000);
|
||||
await resultPromise;
|
||||
|
||||
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toEqual(
|
||||
pendingFinalDelivery("auxiliary\n\ndurable reply", {
|
||||
deliveries: [
|
||||
{ id: "delivery-auxiliary", state: "delivered" },
|
||||
{ id: "delivery-durable", state: "prepared" },
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not let an older settlement rewrite a newer pending-final intent", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pendingFinalDelivery("older reply", { intentId: "older-intent" }),
|
||||
};
|
||||
@@ -381,10 +624,7 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyResolver: async () =>
|
||||
setReplyPayloadMetadata(
|
||||
{ text: "older reply" },
|
||||
{ pendingFinalDeliveryIntentId: "older-intent" },
|
||||
),
|
||||
pendingFinalReply("older reply", { intentId: "older-intent" }),
|
||||
}),
|
||||
});
|
||||
await hookStarted.promise;
|
||||
@@ -407,6 +647,107 @@ describe("dispatchReplyFromConfig reply_dispatch hook", () => {
|
||||
}
|
||||
});
|
||||
|
||||
const createNoSendFailure = (retryable = true) =>
|
||||
new PlatformMessageNotDispatchedError("offline", { cause: new Error("offline"), retryable });
|
||||
const wrapDeliveryFailure = (cause: unknown) =>
|
||||
new OutboundDeliveryError("delivery failed", { cause });
|
||||
const refused = Object.assign(new Error(), {
|
||||
code: "ECONNREFUSED",
|
||||
syscall: "connect",
|
||||
});
|
||||
const createPartialDelivery = () =>
|
||||
Object.assign(new Error("partial delivery", { cause: createNoSendFailure() }), {
|
||||
code: "CHANNEL_PARTIAL_DELIVERY",
|
||||
deliveryResult: { visibleReplySent: true },
|
||||
});
|
||||
|
||||
it.each([
|
||||
["direct retryable provider proof", createNoSendFailure(), true],
|
||||
["wrapped retryable provider proof", wrapDeliveryFailure(createNoSendFailure()), true],
|
||||
["wrapped pre-connect ECONNREFUSED proof", wrapDeliveryFailure(refused), true],
|
||||
["permanent provider rejection", createNoSendFailure(false), false],
|
||||
[
|
||||
"partial outbound delivery",
|
||||
Object.assign(wrapDeliveryFailure(createNoSendFailure()), { sentBeforeError: true }),
|
||||
false,
|
||||
],
|
||||
["nested partial envelope", new Error("partial", { cause: createPartialDelivery() }), false],
|
||||
["aggregate partial envelope", new AggregateError([createPartialDelivery()]), false],
|
||||
["observer-attached delivery evidence", createNoSendFailure(), true],
|
||||
["ambiguous transport failure", new Error("transport failed"), false],
|
||||
] as const)("reconciles pending final delivery after %s", async (name, error, preserve) => {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
const pending = pendingFinalDelivery("recoverable final reply");
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pending,
|
||||
};
|
||||
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
|
||||
existing: sessionStoreMocks.currentEntry,
|
||||
});
|
||||
const dispatcher = createReplyDispatcher({
|
||||
deliver: async () => {
|
||||
throw error;
|
||||
},
|
||||
onError: () => {
|
||||
if (name.startsWith("observer")) {
|
||||
Object.assign(error, { visibleReplySent: true });
|
||||
}
|
||||
},
|
||||
});
|
||||
await withReplyDispatcher({
|
||||
dispatcher,
|
||||
run: () =>
|
||||
dispatchReplyFromConfig({
|
||||
ctx: createHookCtx(),
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyResolver: async () => pendingFinalReply("recoverable final reply"),
|
||||
}),
|
||||
});
|
||||
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toMatchObject({
|
||||
...pending,
|
||||
deliveries: [{ id: "delivery-1", state: preserve ? "prepared" : "unknown" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("clears pending final delivery after intentional pre-delivery cancellation", async () => {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
sessionStoreMocks.currentEntry = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:test:session",
|
||||
pendingFinalDelivery: pendingFinalDelivery("policy-suppressed reply"),
|
||||
};
|
||||
sessionStoreMocks.resolveSessionStoreEntry.mockReturnValue({
|
||||
existing: sessionStoreMocks.currentEntry,
|
||||
});
|
||||
const deliver = vi.fn().mockResolvedValue(undefined);
|
||||
const dispatcher = createReplyDispatcher({
|
||||
deliver,
|
||||
beforeDeliver: () => null,
|
||||
});
|
||||
|
||||
const result = await dispatchReplyFromConfig({
|
||||
ctx: createHookCtx(),
|
||||
cfg: emptyConfig,
|
||||
dispatcher,
|
||||
replyResolver: async () => pendingFinalReply("policy-suppressed reply"),
|
||||
});
|
||||
await dispatcher.waitForIdle();
|
||||
await vi.waitFor(() => {
|
||||
expect(sessionStoreMocks.currentEntry?.pendingFinalDelivery).toBeUndefined();
|
||||
});
|
||||
|
||||
expect(result.queuedFinal).toBe(true);
|
||||
expect(deliver).not.toHaveBeenCalled();
|
||||
// createHookCtx's "private" chat type is undirected, so the cancelled final
|
||||
// does not trigger a fallback attempt.
|
||||
expect(dispatcher.getCancelledCounts?.()).toEqual({ tool: 0, block: 0, final: 1 });
|
||||
expect(dispatcher.getFailedCounts?.()).toEqual({ tool: 0, block: 0, final: 0 });
|
||||
expect(sessionStoreMocks.updateSessionEntry).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("delivers a generated final reply before queued follow-up admission", async () => {
|
||||
hookMocks.runner.hasHooks.mockReturnValue(false);
|
||||
const dispatcher = createDispatcher();
|
||||
|
||||
@@ -4,10 +4,10 @@ import {
|
||||
INTERNAL_RUNTIME_CONTEXT_BEGIN,
|
||||
INTERNAL_RUNTIME_CONTEXT_END,
|
||||
} from "../../agents/internal-runtime-context.js";
|
||||
import { setReplyPayloadMetadata } from "../reply-payload.js";
|
||||
import { markInboundContextLabel } from "./inbound-context-marker.js";
|
||||
import {
|
||||
buildRecoverablePendingFinalDeliveryText,
|
||||
buildPendingFinalDeliveryText,
|
||||
normalizePendingFinalDeliveryPayloads,
|
||||
normalizePendingFinalRecoveryPayloads,
|
||||
sanitizePendingFinalDeliveryText,
|
||||
@@ -95,12 +95,12 @@ describe("normalizePendingFinalRecoveryPayloads", () => {
|
||||
const rawPayloads = [{ text: "Rendered chart\nMEDIA:/tmp/chart.png" }];
|
||||
|
||||
const recoveryPayloads = normalizePendingFinalRecoveryPayloads(rawPayloads);
|
||||
expect(buildPendingFinalDeliveryText(recoveryPayloads)).toBe(
|
||||
expect(recoveryPayloads.map((payload) => payload.text)).toEqual([
|
||||
"Rendered chart\nMEDIA:/tmp/chart.png",
|
||||
);
|
||||
]);
|
||||
|
||||
const deliveryPayloads = normalizePendingFinalDeliveryPayloads(rawPayloads);
|
||||
expect(buildPendingFinalDeliveryText(deliveryPayloads)).toBe("Rendered chart");
|
||||
expect(deliveryPayloads.map((payload) => payload.text)).toEqual(["Rendered chart"]);
|
||||
});
|
||||
|
||||
it("keeps media-only directives as durable recovery text", () => {
|
||||
@@ -108,7 +108,7 @@ describe("normalizePendingFinalRecoveryPayloads", () => {
|
||||
{ text: "MEDIA:/tmp/chart.png" },
|
||||
]);
|
||||
|
||||
expect(buildPendingFinalDeliveryText(recoveryPayloads)).toBe("MEDIA:/tmp/chart.png");
|
||||
expect(recoveryPayloads.map((payload) => payload.text)).toEqual(["MEDIA:/tmp/chart.png"]);
|
||||
expect(normalizePendingFinalDeliveryPayloads(recoveryPayloads)).toHaveLength(1);
|
||||
});
|
||||
|
||||
@@ -139,6 +139,18 @@ describe("normalizePendingFinalRecoveryPayloads", () => {
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("separates implicit delivery threading from explicit reply semantics", () => {
|
||||
expect(
|
||||
buildRecoverablePendingFinalDeliveryText([
|
||||
{ text: "Visible final", replyToId: "source-message" },
|
||||
]),
|
||||
).toBe("Visible final");
|
||||
|
||||
const explicitReply = { text: "Visible final", replyToId: "source-message" };
|
||||
setReplyPayloadMetadata(explicitReply, { replyToIdExplicit: true });
|
||||
expect(buildRecoverablePendingFinalDeliveryText([explicitReply])).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses multi-payload media finals because text recovery loses payload boundaries", () => {
|
||||
expect(
|
||||
buildRecoverablePendingFinalDeliveryText([
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import type { SessionEntry } from "../../config/sessions/types.js";
|
||||
import type { DurableDeliveryCompletion } from "../../infra/outbound/delivery-completion.js";
|
||||
import { normalizeReplyPayloadsForDelivery } from "../../infra/outbound/payloads.js";
|
||||
import { getReplyPayloadMetadata, type ReplyPayload } from "../reply-payload.js";
|
||||
import {
|
||||
isSilentReplyPayloadText,
|
||||
isSilentReplyText,
|
||||
@@ -8,7 +10,6 @@ import {
|
||||
stripLeadingSilentToken,
|
||||
stripSilentToken,
|
||||
} from "../tokens.js";
|
||||
import type { ReplyPayload } from "../types.js";
|
||||
import { stripInternalMetadataForDisplay } from "./display-text-sanitize.js";
|
||||
import { normalizeReplyPayload } from "./normalize-reply.js";
|
||||
|
||||
@@ -38,12 +39,16 @@ export function buildRecoverablePendingFinalDeliveryText(
|
||||
if (payload.isReasoning === true) {
|
||||
continue;
|
||||
}
|
||||
const deliveryPayloads = normalizeReplyPayloadsForDelivery([payload]);
|
||||
const recoveryPayload =
|
||||
payload.replyToId && getReplyPayloadMetadata(payload)?.replyToIdExplicit !== true
|
||||
? { ...payload, replyToId: undefined }
|
||||
: payload;
|
||||
const deliveryPayloads = normalizeReplyPayloadsForDelivery([recoveryPayload]);
|
||||
if (deliveryPayloads.length === 0) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
hasUnsupportedDurableRecoveryShape(payload) ||
|
||||
hasUnsupportedDurableRecoveryShape(recoveryPayload) ||
|
||||
deliveryPayloads.some(hasUnrecoverableNormalizedDeliveryShape)
|
||||
) {
|
||||
return undefined;
|
||||
@@ -78,7 +83,7 @@ export function buildRecoverablePendingFinalDeliveryText(
|
||||
}
|
||||
|
||||
/** Build the restart-recovery text represented by one or more final payloads. */
|
||||
export function buildPendingFinalDeliveryText(payloads: ReplyPayload[]): string {
|
||||
function buildPendingFinalDeliveryText(payloads: ReplyPayload[]): string {
|
||||
const text = payloads
|
||||
.filter((payload) => payload.isReasoning !== true)
|
||||
.map((payload) => payload.text)
|
||||
@@ -93,6 +98,15 @@ export const PENDING_FINAL_DELIVERY_CLEAR_PATCH = {
|
||||
pendingFinalDelivery: undefined,
|
||||
} as const satisfies Partial<SessionEntry>;
|
||||
|
||||
export function resolvePendingFinalDeliveryCompletion(
|
||||
payloads: readonly ReplyPayload[] | undefined,
|
||||
): Extract<DurableDeliveryCompletion, { kind: "pending-final" }> | undefined {
|
||||
const completion = payloads
|
||||
?.map((payload) => getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion)
|
||||
.find(Boolean);
|
||||
return completion ? { kind: "pending-final", ...completion } : undefined;
|
||||
}
|
||||
|
||||
function collectDurableMediaDirectives(payload: ReplyPayload): string[] {
|
||||
if (payload.sensitiveMedia === true) {
|
||||
return [];
|
||||
@@ -122,8 +136,8 @@ function hasUnsupportedDurableRecoveryShape(payload: ReplyPayload): boolean {
|
||||
payload.channelData !== undefined ||
|
||||
payload.location !== undefined ||
|
||||
payload.replyToId !== undefined ||
|
||||
payload.replyToTag !== undefined ||
|
||||
payload.replyToCurrent !== undefined ||
|
||||
payload.replyToTag === true ||
|
||||
payload.replyToCurrent === true ||
|
||||
payload.audioAsVoice === true ||
|
||||
payload.videoAsNote === true ||
|
||||
payload.spokenText !== undefined ||
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
isProvenDeliveryNotSentError,
|
||||
} from "../../infra/delivery-recovery.shared.js";
|
||||
import { collectErrorGraphCandidates } from "../../infra/errors.js";
|
||||
import { settlePendingFinalDelivery } from "../../infra/outbound/delivery-completion.js";
|
||||
import { generateSecureInt } from "../../infra/secure-random.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import type { SilentReplyConversationType } from "../../shared/silent-reply-policy.js";
|
||||
@@ -442,6 +443,7 @@ export function createReplyDispatcher(options: ReplyDispatcherOptions): ReplyDis
|
||||
): Promise<ReplyDispatchDeliveryOutcome> => {
|
||||
let deliverPayload: ReplyPayload | null = payload;
|
||||
let deliveryStarted = false;
|
||||
const custody = getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion;
|
||||
try {
|
||||
if (beforeDeliver) {
|
||||
try {
|
||||
@@ -451,21 +453,60 @@ export function createReplyDispatcher(options: ReplyDispatcherOptions): ReplyDis
|
||||
throw error;
|
||||
}
|
||||
if (!deliverPayload) {
|
||||
// Record the intentional non-delivery before observers run so a
|
||||
// restart during observer work cannot replay a suppressed final.
|
||||
if (custody) {
|
||||
await settlePendingFinalDelivery({ kind: "pending-final", ...custody }, "suppressed", [
|
||||
"prepared",
|
||||
]);
|
||||
}
|
||||
await notifyBeforeDeliverCancelled(payload, info);
|
||||
return "cancelled";
|
||||
}
|
||||
deliverPayload = copyReplyPayloadMetadata(payload, deliverPayload);
|
||||
}
|
||||
if (custody) {
|
||||
// Claim direct-send custody before provider I/O; a non-prepared marker
|
||||
// means another owner already delivered, suppressed, or superseded this
|
||||
// final, so repeating the send would duplicate it.
|
||||
const claim = await settlePendingFinalDelivery(
|
||||
{ kind: "pending-final", ...custody },
|
||||
"queued",
|
||||
["prepared"],
|
||||
);
|
||||
if (claim.state !== "queued") {
|
||||
await notifyBeforeDeliverCancelled(payload, info);
|
||||
return "cancelled";
|
||||
}
|
||||
}
|
||||
deliveryStarted = true;
|
||||
await options.deliver(deliverPayload, info);
|
||||
if (custody) {
|
||||
await settlePendingFinalDelivery({ kind: "pending-final", ...custody }, "delivered", [
|
||||
"queued",
|
||||
]);
|
||||
}
|
||||
return "delivered";
|
||||
} catch (error) {
|
||||
const outcome =
|
||||
deliveryStarted && !isRetryableNoSendFailure(error)
|
||||
? "failed-deliver"
|
||||
: "failed-before-deliver";
|
||||
if (custody && deliveryStarted) {
|
||||
// Proven no-send keeps the marker replayable for restart recovery —
|
||||
// including after direct custody escalated queued→unknown pre-I/O,
|
||||
// since the error proves the send never crossed the wire. Anything
|
||||
// else after platform I/O started fails closed as "unknown".
|
||||
await settlePendingFinalDelivery(
|
||||
{ kind: "pending-final", ...custody },
|
||||
outcome === "failed-deliver" ? "unknown" : "prepared",
|
||||
outcome === "failed-deliver" ? ["queued"] : ["queued", "unknown"],
|
||||
);
|
||||
}
|
||||
try {
|
||||
await options.onError?.(error, info);
|
||||
} catch {}
|
||||
return deliveryStarted && !isRetryableNoSendFailure(error)
|
||||
? "failed-deliver"
|
||||
: "failed-before-deliver";
|
||||
return outcome;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -13,6 +13,10 @@ export type ReplyFollowupAdmissionBarrierTimeoutPolicy = {
|
||||
export type ReplyDispatchRuntimeInfo = {
|
||||
kind: ReplyDispatchKind;
|
||||
assistantMessageIndex?: number;
|
||||
/** @internal Claim direct-send custody immediately before recipient-visible platform I/O. */
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
/** @internal Bind this delivery's host-owned completion to a transformed payload. */
|
||||
bindPendingFinalDelivery?: <T extends ReplyPayload>(payload: T) => T;
|
||||
};
|
||||
|
||||
export type ReplyDispatchBeforeDeliver = (
|
||||
|
||||
@@ -29,6 +29,7 @@ type ReplyRestartRecoveryClaimController = {
|
||||
state: Exclude<RestartRecoveryBeforeAgentReplyState, "admitted" | "pending">;
|
||||
pendingFinalDelivery?: {
|
||||
context?: DeliveryContext;
|
||||
deliveries: NonNullable<SessionEntry["pendingFinalDelivery"]>["deliveries"];
|
||||
intentId: string;
|
||||
text: string;
|
||||
};
|
||||
@@ -376,6 +377,7 @@ export function createReplyRestartRecoveryClaimController(params: {
|
||||
...(pendingFinalDelivery.intentId
|
||||
? { intentId: pendingFinalDelivery.intentId }
|
||||
: {}),
|
||||
deliveries: pendingFinalDelivery.deliveries,
|
||||
...(pendingFinalDelivery.context
|
||||
? { context: pendingFinalDelivery.context }
|
||||
: {}),
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
* Sends rendered reply payloads, records live preview state, and classifies delivery outcomes.
|
||||
*/
|
||||
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
|
||||
import { resolvePendingFinalDeliveryCompletion } from "../../auto-reply/reply/pending-final-delivery.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import type { OutboundDeliveryResult } from "../../infra/outbound/deliver-types.js";
|
||||
import {
|
||||
@@ -395,14 +396,27 @@ export async function withDurableMessageSendContext<T>(
|
||||
export async function sendDurableMessageBatch(
|
||||
params: DurableMessageSendContextParams,
|
||||
): Promise<DurableMessageBatchSendResult> {
|
||||
return await withDurableMessageSendContext(params, async (ctx) => {
|
||||
const rendered = await ctx.render();
|
||||
const result = await ctx.send(rendered);
|
||||
if (result.status === "sent" || result.status === "suppressed") {
|
||||
await ctx.commit(result.receipt);
|
||||
} else {
|
||||
await ctx.fail(result.error);
|
||||
}
|
||||
return result;
|
||||
});
|
||||
const pendingFinalCompletion = params.deliveryCompletion
|
||||
? undefined
|
||||
: resolvePendingFinalDeliveryCompletion(params.payloads);
|
||||
const pendingFinalDelivery = pendingFinalCompletion
|
||||
? {
|
||||
deliveryCompletion: pendingFinalCompletion,
|
||||
deliveryIntentId: pendingFinalCompletion.deliveryId,
|
||||
durability: "required" as const,
|
||||
}
|
||||
: {};
|
||||
return await withDurableMessageSendContext(
|
||||
{ ...params, ...pendingFinalDelivery },
|
||||
async (ctx) => {
|
||||
const rendered = await ctx.render();
|
||||
const result = await ctx.send(rendered);
|
||||
if (result.status === "sent" || result.status === "suppressed") {
|
||||
await ctx.commit(result.receipt);
|
||||
} else {
|
||||
await ctx.fail(result.error);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -832,6 +832,8 @@ export type ChannelPollContext = {
|
||||
silent?: boolean;
|
||||
isAnonymous?: boolean;
|
||||
gatewayClientScopes?: readonly string[];
|
||||
/** @internal Refresh durable timing before recipient-visible platform I/O. */
|
||||
onPlatformSendDispatch?: () => Promise<void>;
|
||||
};
|
||||
|
||||
/** Minimal base for all channel probe results. Channel-specific probes extend this. */
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import {
|
||||
getReplyPayloadMetadata,
|
||||
setReplyPayloadMetadata,
|
||||
type ReplyPayload,
|
||||
} from "../../auto-reply/reply-payload.js";
|
||||
import { PlatformMessageNotDispatchedError } from "../../infra/outbound/deliver-types.js";
|
||||
import { settlePendingFinalDelivery } from "../../infra/outbound/delivery-completion.js";
|
||||
import type { ChannelDeliveryInfo } from "./types.js";
|
||||
|
||||
type DirectPendingFinalCustody = Pick<ChannelDeliveryInfo, "bindPendingFinalDelivery"> & {
|
||||
onPlatformSendDispatch: () => Promise<void>;
|
||||
};
|
||||
|
||||
export const NO_PENDING_FINAL_CUSTODY: DirectPendingFinalCustody = {
|
||||
onPlatformSendDispatch: () => Promise.resolve(),
|
||||
};
|
||||
|
||||
export function resolvePendingFinalCompletion(payload: ReplyPayload) {
|
||||
const identity = getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion;
|
||||
return identity ? { kind: "pending-final" as const, ...identity } : undefined;
|
||||
}
|
||||
|
||||
export function createDirectPendingFinalCustody(
|
||||
payload: ReplyPayload,
|
||||
): DirectPendingFinalCustody | undefined {
|
||||
const completion = resolvePendingFinalCompletion(payload);
|
||||
if (!completion) {
|
||||
return undefined;
|
||||
}
|
||||
const { kind: _kind, ...identity } = completion;
|
||||
let admission: Promise<void> | undefined;
|
||||
return {
|
||||
bindPendingFinalDelivery: (nextPayload) =>
|
||||
setReplyPayloadMetadata(nextPayload, {
|
||||
pendingFinalDeliveryCompletion: identity,
|
||||
}),
|
||||
onPlatformSendDispatch: () => {
|
||||
admission ??= settlePendingFinalDelivery(completion, "unknown", ["prepared", "queued"]).then(
|
||||
(result) => {
|
||||
if (result.state !== "unknown") {
|
||||
throw new PlatformMessageNotDispatchedError(
|
||||
"Pending final delivery ownership changed before platform dispatch",
|
||||
{ cause: new Error(`pending final delivery is ${result.state}`) },
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
return admission;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function toCoreManagedDeliveryInfo(info: ChannelDeliveryInfo) {
|
||||
return {
|
||||
kind: info.kind,
|
||||
...(info.assistantMessageIndex === undefined
|
||||
? {}
|
||||
: { assistantMessageIndex: info.assistantMessageIndex }),
|
||||
};
|
||||
}
|
||||
@@ -204,7 +204,6 @@ export async function deliverInboundReplyWithMessageSendContext(
|
||||
requesterSenderUsername: params.ctxPayload.SenderUsername,
|
||||
requesterSenderE164: params.ctxPayload.SenderE164,
|
||||
});
|
||||
|
||||
const send = await sendDurableMessageBatch({
|
||||
cfg: params.cfg,
|
||||
channel,
|
||||
@@ -220,7 +219,9 @@ export async function deliverInboundReplyWithMessageSendContext(
|
||||
mediaAccess: params.mediaAccess,
|
||||
silent: params.silent,
|
||||
durability,
|
||||
...(durability === "required" ? { requireUnknownSendReconciliation: true } : {}),
|
||||
...(requiredCapabilities.reconcileUnknownSend === true
|
||||
? { requireUnknownSendReconciliation: true }
|
||||
: {}),
|
||||
session,
|
||||
gatewayClientScopes: params.ctxPayload.GatewayClientScopes ?? [],
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { dispatchInboundMessageWithRoutedChannelDispatcher } from "../../auto-reply/dispatch.js";
|
||||
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
|
||||
import { copyReplyPayloadMetadata, type ReplyPayload } from "../../auto-reply/reply-payload.js";
|
||||
import { suppressPendingFinalDelivery } from "../../auto-reply/reply/dispatch-from-config.pending-final.js";
|
||||
import type { DispatchFromConfigResult } from "../../auto-reply/reply/dispatch-from-config.types.js";
|
||||
import type { ReplyDispatchKind } from "../../auto-reply/reply/reply-dispatcher.types.js";
|
||||
import { runWithSessionInitConflictRetry } from "../../auto-reply/reply/session-init-conflict-retry.js";
|
||||
@@ -12,6 +13,7 @@ import { formatErrorMessage, toErrorObject } from "../../infra/errors.js";
|
||||
import { applyMessageSendingHook } from "../../infra/outbound/deliver-hooks.js";
|
||||
import { normalizeEmptyPayloadForDelivery } from "../../infra/outbound/deliver-payload.js";
|
||||
import { isPlatformMessageNotDispatchedError } from "../../infra/outbound/deliver-types.js";
|
||||
import { settlePendingFinalDelivery } from "../../infra/outbound/delivery-completion.js";
|
||||
import { createMessageSentEmitter } from "../../infra/outbound/message-sent-hook.js";
|
||||
import { summarizeOutboundPayloadForTransport } from "../../infra/outbound/payloads.js";
|
||||
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
|
||||
@@ -19,6 +21,12 @@ import { resolveMessageReceiptPrimaryId } from "../message/receipt.js";
|
||||
import { createChannelReplyPipeline } from "../message/reply-pipeline.js";
|
||||
import { recordInboundSession } from "../session.js";
|
||||
import { isChannelPartialDeliveryError } from "./delivery-result.js";
|
||||
import {
|
||||
createDirectPendingFinalCustody,
|
||||
NO_PENDING_FINAL_CUSTODY,
|
||||
resolvePendingFinalCompletion,
|
||||
toCoreManagedDeliveryInfo,
|
||||
} from "./direct-delivery-custody.js";
|
||||
import {
|
||||
deliverInboundReplyWithMessageSendContext,
|
||||
isDurableInboundReplyDeliveryHandled,
|
||||
@@ -266,6 +274,13 @@ async function settleChannelDeliveryAttempt(params: {
|
||||
messageId: resolveChannelDeliveryMessageId(finalized),
|
||||
});
|
||||
}
|
||||
const completion = resolvePendingFinalCompletion(attempt.payload);
|
||||
if (completion) {
|
||||
await settlePendingFinalDelivery(
|
||||
completion,
|
||||
isExplicitlyNonVisibleChannelDelivery(finalized) ? "suppressed" : "delivered",
|
||||
);
|
||||
}
|
||||
await runChannelDeliveryObserver({
|
||||
onDelivered: params.onDelivered,
|
||||
payload: attempt.payload,
|
||||
@@ -332,7 +347,7 @@ async function applyRoutedDirectMessageSending(params: {
|
||||
}),
|
||||
};
|
||||
}
|
||||
return { payload };
|
||||
return { payload: copyReplyPayloadMetadata(params.payload, payload) };
|
||||
}
|
||||
|
||||
function reconcileNonVisibleChannelDeliveries(
|
||||
@@ -465,6 +480,7 @@ async function dispatchChannelTurnWithDeliveryOwner(
|
||||
| "cancelled_by_reply_payload_sending_hook"
|
||||
| "empty_after_reply_payload_sending_hook",
|
||||
) => {
|
||||
await suppressPendingFinalDelivery(payload);
|
||||
await runChannelDeliveryObserver({
|
||||
onDelivered: delivery.onDelivered,
|
||||
payload,
|
||||
@@ -477,13 +493,18 @@ async function dispatchChannelTurnWithDeliveryOwner(
|
||||
dispatcherOptions: {
|
||||
...replyPipeline.dispatcherOptions,
|
||||
deliver: async (payload: ReplyPayload, info: ChannelDeliveryInfo) => {
|
||||
const preparedPayload = delivery.preparePayload
|
||||
const preparedPayloadResult = delivery.preparePayload
|
||||
? await delivery.preparePayload(payload, info)
|
||||
: payload;
|
||||
const preparedPayload =
|
||||
preparedPayloadResult === null
|
||||
? null
|
||||
: copyReplyPayloadMetadata(payload, preparedPayloadResult);
|
||||
if (preparedPayload === null) {
|
||||
const suppression = createSuppressedChannelDeliveryResult({
|
||||
reason: "no_visible_payload",
|
||||
});
|
||||
await suppressPendingFinalDelivery(payload);
|
||||
await runChannelDeliveryObserver({
|
||||
onDelivered: delivery.onDelivered,
|
||||
payload,
|
||||
@@ -525,15 +546,22 @@ async function dispatchChannelTurnWithDeliveryOwner(
|
||||
}
|
||||
let effectivePayload = preparedPayload;
|
||||
let result: ChannelDeliveryResult | void = undefined;
|
||||
let directInfo: ChannelDeliveryInfo = info;
|
||||
try {
|
||||
if (
|
||||
ownership === "routed-delivery" &&
|
||||
"deliverWithProviderMessageSending" in delivery &&
|
||||
delivery.deliverWithProviderMessageSending
|
||||
) {
|
||||
const providerInfo = {
|
||||
...info,
|
||||
...(createDirectPendingFinalCustody(effectivePayload) ??
|
||||
NO_PENDING_FINAL_CUSTODY),
|
||||
};
|
||||
directInfo = providerInfo;
|
||||
result = await delivery.deliverWithProviderMessageSending(
|
||||
effectivePayload,
|
||||
info,
|
||||
providerInfo,
|
||||
);
|
||||
} else {
|
||||
if (
|
||||
@@ -555,13 +583,22 @@ async function dispatchChannelTurnWithDeliveryOwner(
|
||||
"channel delivery adapter is missing a direct deliverer",
|
||||
);
|
||||
}
|
||||
result = await delivery.deliver(effectivePayload, info);
|
||||
const custody = createDirectPendingFinalCustody(effectivePayload);
|
||||
await custody?.onPlatformSendDispatch();
|
||||
result = await delivery.deliver(
|
||||
effectivePayload,
|
||||
toCoreManagedDeliveryInfo(info),
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (delivery.observeMessageSent) {
|
||||
await settleChannelDeliveryAttempt({
|
||||
attempt: { payload: effectivePayload, info, error },
|
||||
attempt: {
|
||||
payload: effectivePayload,
|
||||
info: directInfo,
|
||||
error,
|
||||
},
|
||||
onDelivered: delivery.onDelivered,
|
||||
emitMessageSent: getMessageSentEmitter()?.emitMessageSent,
|
||||
});
|
||||
@@ -572,22 +609,24 @@ async function dispatchChannelTurnWithDeliveryOwner(
|
||||
// Finalization can reject while the buffered dispatcher is still unwinding.
|
||||
// Observe it now; settlement still awaits the original promise and its error.
|
||||
void result.finalization.catch(() => undefined);
|
||||
pendingDeliveryAttempts.push({ payload: effectivePayload, info, result });
|
||||
} else if (delivery.observeMessageSent) {
|
||||
const finalized = await settleChannelDeliveryAttempt({
|
||||
attempt: { payload: effectivePayload, info, result },
|
||||
onDelivered: delivery.onDelivered,
|
||||
emitMessageSent: getMessageSentEmitter()?.emitMessageSent,
|
||||
});
|
||||
recordSettledDelivery(info, finalized);
|
||||
} else {
|
||||
await runChannelDeliveryObserver({
|
||||
onDelivered: delivery.onDelivered,
|
||||
pendingDeliveryAttempts.push({
|
||||
payload: effectivePayload,
|
||||
info,
|
||||
info: directInfo,
|
||||
result,
|
||||
});
|
||||
recordSettledDelivery(info, result ?? undefined);
|
||||
} else {
|
||||
const finalized = await settleChannelDeliveryAttempt({
|
||||
attempt: {
|
||||
payload: effectivePayload,
|
||||
info: directInfo,
|
||||
result,
|
||||
},
|
||||
onDelivered: delivery.onDelivered,
|
||||
emitMessageSent: delivery.observeMessageSent
|
||||
? getMessageSentEmitter()?.emitMessageSent
|
||||
: undefined,
|
||||
});
|
||||
recordSettledDelivery(info, finalized);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
// Channel turn delivery tests cover orchestration, dispatch, and completion behavior.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
|
||||
import {
|
||||
getReplyPayloadMetadata,
|
||||
setReplyPayloadMetadata,
|
||||
type ReplyPayload,
|
||||
} from "../../auto-reply/reply-payload.js";
|
||||
import type { DispatchReplyWithBufferedBlockDispatcher } from "../../auto-reply/reply/provider-dispatcher.types.js";
|
||||
import type { FinalizedMsgContext } from "../../auto-reply/templating.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
@@ -10,7 +14,7 @@ import { outboundMessageIdentities } from "../message/outbound-echo-state.js";
|
||||
import type { RecordInboundSession } from "../session.types.js";
|
||||
import { hasVisibleChannelTurnDispatch } from "./dispatch-result.js";
|
||||
import { dispatchAssembledChannelTurn, dispatchRoutedChannelTurn } from "./lifecycle.js";
|
||||
import type { ChannelTurnResult } from "./types.js";
|
||||
import type { ChannelDeliveryInfo, ChannelTurnResult } from "./types.js";
|
||||
|
||||
const deliverOutboundPayloads = vi.hoisted(() => vi.fn());
|
||||
const resolveOutboundDurableFinalDeliverySupport = vi.hoisted(() => vi.fn());
|
||||
@@ -24,6 +28,9 @@ const createMessageSentEmitter = vi.hoisted(() =>
|
||||
vi.fn(() => ({ emitMessageSent, hasMessageSentHooks: true })),
|
||||
);
|
||||
const readRecentUserAssistantTextForSession = vi.hoisted(() => vi.fn());
|
||||
const settlePendingFinalDelivery = vi.hoisted(() =>
|
||||
vi.fn(async (_completion: unknown, state: string) => ({ state })),
|
||||
);
|
||||
|
||||
vi.mock("../../auto-reply/reply/provider-dispatcher.js", async (importOriginal) => {
|
||||
const actual =
|
||||
@@ -77,6 +84,12 @@ vi.mock("../../config/sessions/transcript.js", () => ({
|
||||
readRecentUserAssistantTextForSession,
|
||||
}));
|
||||
|
||||
vi.mock("../../infra/outbound/delivery-completion.js", async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import("../../infra/outbound/delivery-completion.js")>();
|
||||
return { ...actual, settlePendingFinalDelivery };
|
||||
});
|
||||
|
||||
const cfg = {} as OpenClawConfig;
|
||||
|
||||
function createCtx(overrides: Partial<FinalizedMsgContext> = {}): FinalizedMsgContext {
|
||||
@@ -286,6 +299,77 @@ describe("channel turn delivery", () => {
|
||||
expect(result.dispatchResult.counts.final).toBe(1);
|
||||
});
|
||||
|
||||
it("preserves pending final custody through preparation and message hook rewrites", async () => {
|
||||
const order: string[] = [];
|
||||
const completion = {
|
||||
deliveryId: "delivery-1",
|
||||
intentId: "intent-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:main:telegram:peer",
|
||||
storePath: "/tmp/sessions.json",
|
||||
};
|
||||
const sourcePayload = setReplyPayloadMetadata(
|
||||
{ text: "reply" },
|
||||
{ pendingFinalDeliveryCompletion: completion },
|
||||
);
|
||||
dispatchReplyWithRoutedChannelDispatcherCore.mockImplementationOnce(async (params) => {
|
||||
await params.dispatcherOptions.deliver(sourcePayload, { kind: "final" });
|
||||
return { queuedFinal: true, counts: { tool: 0, block: 0, final: 1 } };
|
||||
});
|
||||
getGlobalHookRunner.mockReturnValue({
|
||||
hasHooks: (name: string) => name === "message_sending",
|
||||
runMessageSending: vi.fn(async ({ content }: { content: string }) => ({
|
||||
content: `${content} + hook`,
|
||||
})),
|
||||
});
|
||||
let releaseDelivery: (() => void) | undefined;
|
||||
const deliveryPending = new Promise<void>((resolve) => {
|
||||
releaseDelivery = resolve;
|
||||
});
|
||||
settlePendingFinalDelivery.mockImplementationOnce(async (_completion, state: string) => {
|
||||
order.push(`settle:${state}`);
|
||||
return { state };
|
||||
});
|
||||
const deliver = vi.fn(async (payload: ReplyPayload, info: ChannelDeliveryInfo) => {
|
||||
expect(getReplyPayloadMetadata(payload)?.pendingFinalDeliveryCompletion).toEqual(completion);
|
||||
expect("onPlatformSendDispatch" in info).toBe(false);
|
||||
order.push("signal:accepted");
|
||||
await deliveryPending;
|
||||
return { messageIds: ["direct-1"], visibleReplySent: true };
|
||||
});
|
||||
|
||||
const dispatch = dispatchRoutedChannelTurn({
|
||||
cfg,
|
||||
channel: "telegram",
|
||||
accountId: "acct",
|
||||
route: { agentId: "main", sessionKey: completion.sessionKey },
|
||||
ctxPayload: createCtx({ Surface: "telegram", OriginatingTo: "chat-1" }),
|
||||
delivery: {
|
||||
preparePayload: (payload) => ({ ...payload, text: `${payload.text} + prepared` }),
|
||||
deliver,
|
||||
},
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(deliver).toHaveBeenCalledOnce());
|
||||
expect(order).toEqual(["settle:unknown", "signal:accepted"]);
|
||||
releaseDelivery?.();
|
||||
await dispatch;
|
||||
|
||||
expect(deliver).toHaveBeenCalledOnce();
|
||||
expect(deliver.mock.calls[0]?.[0]).toMatchObject({ text: "reply + prepared + hook" });
|
||||
expect(settlePendingFinalDelivery).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
{ kind: "pending-final", ...completion },
|
||||
"unknown",
|
||||
["prepared", "queued"],
|
||||
);
|
||||
expect(settlePendingFinalDelivery).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
{ kind: "pending-final", ...completion },
|
||||
"delivered",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not let message hooks resurrect payloads suppressed during preparation", async () => {
|
||||
const runMessageSending = vi.fn(async () => ({ content: "resurrected" }));
|
||||
getGlobalHookRunner.mockReturnValue({
|
||||
@@ -454,7 +538,10 @@ describe("channel turn delivery", () => {
|
||||
|
||||
expect(deliverWithProviderMessageSending).toHaveBeenCalledWith(
|
||||
{ text: "reply" },
|
||||
{ kind: "final" },
|
||||
expect.objectContaining({
|
||||
kind: "final",
|
||||
onPlatformSendDispatch: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
expect(runMessageSending).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -9,7 +9,7 @@ import type { GetReplyFromConfig } from "../../auto-reply/reply/get-reply.types.
|
||||
import type { HistoryEntry, HistoryMediaEntry } from "../../auto-reply/reply/history.types.js";
|
||||
import type { DispatchReplyWithBufferedBlockDispatcher } from "../../auto-reply/reply/provider-dispatcher.types.js";
|
||||
import type { ReplyDispatcherWithTypingOptions } from "../../auto-reply/reply/reply-dispatcher.js";
|
||||
import type { ReplyDispatchKind } from "../../auto-reply/reply/reply-dispatcher.types.js";
|
||||
import type { ReplyDispatchRuntimeInfo } from "../../auto-reply/reply/reply-dispatcher.types.js";
|
||||
import type {
|
||||
FinalizedMsgContext,
|
||||
InboundSourceModality,
|
||||
@@ -156,8 +156,15 @@ export type PreflightFacts = {
|
||||
};
|
||||
|
||||
/** Delivery metadata for one reply payload dispatch. */
|
||||
export type ChannelDeliveryInfo = {
|
||||
kind: ReplyDispatchKind;
|
||||
export type ChannelDeliveryInfo = ReplyDispatchRuntimeInfo;
|
||||
|
||||
type ChannelCoreManagedDeliveryInfo = Omit<
|
||||
ChannelDeliveryInfo,
|
||||
"bindPendingFinalDelivery" | "onPlatformSendDispatch"
|
||||
>;
|
||||
|
||||
type ChannelProviderOwnedDeliveryInfo = ChannelDeliveryInfo & {
|
||||
onPlatformSendDispatch: () => Promise<void>;
|
||||
};
|
||||
|
||||
/** Durable delivery queue intent recorded when a reply is deferred. */
|
||||
@@ -220,7 +227,7 @@ type ChannelDeliveryAdapterBase = {
|
||||
export type ChannelCoreManagedTurnDeliveryAdapter = ChannelDeliveryAdapterBase & {
|
||||
deliver: (
|
||||
payload: ReplyPayload,
|
||||
info: ChannelDeliveryInfo,
|
||||
info: ChannelCoreManagedDeliveryInfo,
|
||||
) => Promise<ChannelDeliveryResult | void>;
|
||||
durable?:
|
||||
| false
|
||||
@@ -245,7 +252,7 @@ export type ChannelProviderOwnedMessageSendingDeliveryAdapter = ChannelDeliveryA
|
||||
*/
|
||||
deliverWithProviderMessageSending: (
|
||||
payload: ReplyPayload,
|
||||
info: ChannelDeliveryInfo,
|
||||
info: ChannelProviderOwnedDeliveryInfo,
|
||||
) => Promise<ChannelDeliveryResult | void>;
|
||||
deliver?: never;
|
||||
durable?: never;
|
||||
|
||||
@@ -85,6 +85,7 @@ vi.mock("../agents/command/session-store.runtime.js", async () => {
|
||||
const accessor = await import("../config/sessions/session-accessor.js");
|
||||
return {
|
||||
loadSessionEntry: accessor.loadSessionEntry,
|
||||
loadSessionEntryReadOnly: accessor.loadSessionEntryReadOnly,
|
||||
updateSessionStoreAfterAgentRun: vi.fn(async () => undefined),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -81,6 +81,35 @@ it("normalizes boolean-only pending delivery as transport-only", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes exact pending-final delivery owners", () => {
|
||||
expect(
|
||||
normalizePersistedSessionEntryShape({
|
||||
sessionId: "session-1",
|
||||
updatedAt: 42,
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: "durable reply",
|
||||
createdAt: 41,
|
||||
intentId: "intent-1",
|
||||
deliveries: [
|
||||
{ id: "delivery-prepared", state: "prepared" },
|
||||
{ id: "delivery-delivered", state: "delivered" },
|
||||
{ id: "", state: "queued" },
|
||||
{ id: "delivery-invalid", state: "invalid" },
|
||||
],
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
pendingFinalDelivery: {
|
||||
intentId: "intent-1",
|
||||
deliveries: [
|
||||
{ id: "delivery-prepared", state: "prepared" },
|
||||
{ id: "delivery-delivered", state: "delivered" },
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes and preserves the durable assistant transcript repair backlog", () => {
|
||||
expect(
|
||||
normalizePersistedSessionEntryShape({
|
||||
|
||||
@@ -144,10 +144,30 @@ function normalizePendingFinalDelivery(
|
||||
return undefined;
|
||||
}
|
||||
const intentId = normalizeOptionalString(value.intentId);
|
||||
const deliveries: NonNullable<SessionEntry["pendingFinalDelivery"]>["deliveries"] = Array.isArray(
|
||||
value.deliveries,
|
||||
)
|
||||
? value.deliveries.flatMap((delivery) => {
|
||||
if (!isRecord(delivery)) {
|
||||
return [];
|
||||
}
|
||||
const id = normalizeOptionalString(delivery.id);
|
||||
const state = delivery.state;
|
||||
return id &&
|
||||
(state === "prepared" ||
|
||||
state === "queued" ||
|
||||
state === "delivered" ||
|
||||
state === "suppressed" ||
|
||||
state === "unknown")
|
||||
? [{ id, state }]
|
||||
: [];
|
||||
})
|
||||
: undefined;
|
||||
const base = {
|
||||
createdAt,
|
||||
...(isRecord(value.context) ? { context: value.context } : {}),
|
||||
...(intentId ? { intentId } : {}),
|
||||
...(deliveries ? { deliveries } : {}),
|
||||
};
|
||||
if (value.kind === "transport-only") {
|
||||
return { kind: "transport-only", ...base };
|
||||
|
||||
@@ -66,6 +66,10 @@ type PendingFinalDeliveryState = {
|
||||
createdAt: number;
|
||||
context?: DeliveryContext;
|
||||
intentId?: string;
|
||||
deliveries?: Array<{
|
||||
id: string;
|
||||
state: "prepared" | "queued" | "delivered" | "suppressed" | "unknown";
|
||||
}>;
|
||||
} & ({ kind: "replayable"; text: string } | { kind: "transport-only" });
|
||||
|
||||
/**
|
||||
|
||||
@@ -303,9 +303,13 @@ export async function deliverOutboundPayloadsWithQueueCleanup(
|
||||
if (!queueId) {
|
||||
if (params.deliveryCompletion) {
|
||||
if (results.length > 0) {
|
||||
completeDurableDelivery(params.deliveryCompletion, results.at(-1)!);
|
||||
await completeDurableDelivery(
|
||||
params.deliveryCompletion,
|
||||
results.at(-1)!,
|
||||
platformQueueStateDir,
|
||||
);
|
||||
} else {
|
||||
suppressDurableDelivery(params.deliveryCompletion);
|
||||
await suppressDurableDelivery(params.deliveryCompletion, platformQueueStateDir);
|
||||
}
|
||||
}
|
||||
if (!params.deferCommitHooks) {
|
||||
@@ -363,9 +367,13 @@ export async function deliverOutboundPayloadsWithQueueCleanup(
|
||||
} else {
|
||||
if (params.deliveryCompletion) {
|
||||
if (results.length > 0) {
|
||||
completeDurableDelivery(params.deliveryCompletion, results.at(-1)!);
|
||||
await completeDurableDelivery(
|
||||
params.deliveryCompletion,
|
||||
results.at(-1)!,
|
||||
platformQueueStateDir,
|
||||
);
|
||||
} else {
|
||||
suppressDurableDelivery(params.deliveryCompletion);
|
||||
await suppressDurableDelivery(params.deliveryCompletion, platformQueueStateDir);
|
||||
}
|
||||
}
|
||||
const postSendState =
|
||||
@@ -562,7 +570,11 @@ export async function deliverOutboundPayloadsWithQueueCleanup(
|
||||
terminalRejectionHandled = true;
|
||||
} else {
|
||||
if (params.deliveryCompletion) {
|
||||
rejectDurableDelivery(params.deliveryCompletion, permanentRejection.message);
|
||||
await rejectDurableDelivery(
|
||||
params.deliveryCompletion,
|
||||
permanentRejection.message,
|
||||
platformQueueStateDir,
|
||||
);
|
||||
ownerRejected = true;
|
||||
}
|
||||
await (producerClaimId
|
||||
|
||||
@@ -14,7 +14,9 @@ import {
|
||||
stageAndEnqueueOutboundDelivery,
|
||||
} from "./deliver-queue-admission.js";
|
||||
import { deliverOutboundPayloadsWithQueueCleanup } from "./deliver-queue-execute.js";
|
||||
import { createQueuedDeliveryOwner } from "./deliver-queue-state.js";
|
||||
import type { OutboundDeliveryResult } from "./deliver-types.js";
|
||||
import { markDurableDeliveryQueued } from "./delivery-completion.js";
|
||||
import { startDeliveryProducerLease } from "./delivery-queue-lease.js";
|
||||
import {
|
||||
StableDeliveryPreparationLostError,
|
||||
@@ -272,6 +274,20 @@ async function runOutboundDeliveryWithQueue(
|
||||
if (queued?.created && stablePreparationOwner) {
|
||||
stablePreparationOwner.markPublished();
|
||||
}
|
||||
if (queueId && params.deliveryCompletion) {
|
||||
const completion = await markDurableDeliveryQueued(
|
||||
params.deliveryCompletion,
|
||||
queueId,
|
||||
queued?.created ? "prepared" : undefined,
|
||||
);
|
||||
if (completion.state !== "queued") {
|
||||
await createQueuedDeliveryOwner({
|
||||
queueId,
|
||||
expectedPlatformSendAttemptId: queued?.producerClaimId,
|
||||
}).ack({ suppressCompletionReceipt: true });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
if (queueId) {
|
||||
params.onDeliveryIntent?.({
|
||||
id: queueId,
|
||||
|
||||
@@ -96,6 +96,7 @@ const queueMocks = vi.hoisted(() => ({
|
||||
}));
|
||||
const completionMocks = vi.hoisted(() => ({
|
||||
completeDurableDelivery: vi.fn(),
|
||||
markDurableDeliveryQueued: vi.fn(async () => ({ state: "queued" as const })),
|
||||
rejectDurableDelivery: vi.fn(),
|
||||
suppressDurableDelivery: vi.fn(),
|
||||
}));
|
||||
@@ -207,6 +208,7 @@ vi.mock("./delivery-queue.js", () => ({
|
||||
}));
|
||||
vi.mock("./delivery-completion.js", () => ({
|
||||
completeDurableDelivery: completionMocks.completeDurableDelivery,
|
||||
markDurableDeliveryQueued: completionMocks.markDurableDeliveryQueued,
|
||||
rejectDurableDelivery: completionMocks.rejectDurableDelivery,
|
||||
suppressDurableDelivery: completionMocks.suppressDurableDelivery,
|
||||
}));
|
||||
@@ -521,6 +523,7 @@ describe("deliverOutboundPayloads", () => {
|
||||
},
|
||||
);
|
||||
completionMocks.completeDurableDelivery.mockClear();
|
||||
completionMocks.markDurableDeliveryQueued.mockClear();
|
||||
completionMocks.rejectDurableDelivery.mockClear();
|
||||
completionMocks.suppressDurableDelivery.mockClear();
|
||||
queueMocks.ackDelivery.mockClear();
|
||||
@@ -932,6 +935,26 @@ describe("deliverOutboundPayloads", () => {
|
||||
expect(results[0]?.messageId).toBe("message-adapter-1");
|
||||
});
|
||||
|
||||
it("does not claim platform custody when message adapter preflight fails", async () => {
|
||||
const messageSendText = vi.fn();
|
||||
setMatrixMessageAdapter({
|
||||
id: "matrix",
|
||||
durableFinal: { capabilities: { text: true } },
|
||||
send: {
|
||||
lifecycle: {
|
||||
beforeSendAttempt: () => {
|
||||
throw new Error("preflight rejected");
|
||||
},
|
||||
},
|
||||
text: messageSendText,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(deliverMatrix({ queuePolicy: "required" })).rejects.toThrow("preflight rejected");
|
||||
expect(queueMocks.markDeliveryPlatformSendDispatched).not.toHaveBeenCalled();
|
||||
expect(messageSendText).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not cross platform I/O when a stable queue intent already exists", async () => {
|
||||
hookMocks.runner.hasHooks.mockImplementation((name?: string) => name === "message_sending");
|
||||
queueMocks.findDeliveryIntentOwner.mockReturnValue({
|
||||
@@ -1063,6 +1086,7 @@ describe("deliverOutboundPayloads", () => {
|
||||
expect(completionMocks.completeDurableDelivery).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ operationId: "operation-chunked" }),
|
||||
expect.objectContaining({ messageId: "chunk-2" }),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -2169,6 +2193,7 @@ describe("deliverOutboundPayloads", () => {
|
||||
expect(completionMocks.rejectDurableDelivery).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ operationId: "operation-rejected" }),
|
||||
"atomic message limit",
|
||||
undefined,
|
||||
);
|
||||
expect(queueMocks.failDeliveryBeforePlatformSend).not.toHaveBeenCalled();
|
||||
expect(queueMocks.failDelivery).not.toHaveBeenCalled();
|
||||
@@ -2206,6 +2231,7 @@ describe("deliverOutboundPayloads", () => {
|
||||
expect(completionMocks.rejectDurableDelivery).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ operationId: "operation-empty-rejection" }),
|
||||
"Platform rejected the message before dispatch",
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import { createEmptyPluginRegistry } from "../../plugins/registry.js";
|
||||
import { resetPluginRuntimeStateForTest, setActivePluginRegistry } from "../../plugins/runtime.js";
|
||||
import { createOutboundTestPlugin, createTestRegistry } from "../../test-utils/channel-plugins.js";
|
||||
import { matrixOutboundForQueueTest } from "./deliver.queue-integration.test-support.js";
|
||||
import { loadPendingDeliveries } from "./delivery-queue-storage.js";
|
||||
import { installDeliveryQueueTmpDirHooks } from "./delivery-queue.test-helpers.js";
|
||||
|
||||
let deliverOutboundPayloads: typeof import("./deliver.js").deliverOutboundPayloads;
|
||||
|
||||
describe("pending-final durable delivery completion", () => {
|
||||
const fixtures = installDeliveryQueueTmpDirHooks();
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ deliverOutboundPayloads } = await import("./deliver.js"));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fixtures.tmpDir();
|
||||
setActivePluginRegistry(
|
||||
createTestRegistry([
|
||||
{
|
||||
pluginId: "matrix",
|
||||
source: "test",
|
||||
plugin: createOutboundTestPlugin({ id: "matrix", outbound: matrixOutboundForQueueTest }),
|
||||
},
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetPluginRuntimeStateForTest();
|
||||
setActivePluginRegistry(createEmptyPluginRegistry());
|
||||
});
|
||||
|
||||
it("suppresses a second stable caller after the exact pending final was delivered", async () => {
|
||||
process.env.OPENCLAW_STATE_DIR = tmpDir;
|
||||
const sessionKey = "agent:main:matrix:direct:123";
|
||||
const storePath = path.join(tmpDir, "sessions.json");
|
||||
const deliveryId = "pending-final-delivery-1";
|
||||
const completion = {
|
||||
kind: "pending-final" as const,
|
||||
deliveryId,
|
||||
intentId: "pending-final-intent-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
storePath,
|
||||
};
|
||||
await replaceSessionEntry(
|
||||
{ sessionKey, storePath },
|
||||
{
|
||||
sessionId: "session-1",
|
||||
status: "running",
|
||||
updatedAt: Date.now(),
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: "deliver once",
|
||||
createdAt: Date.now(),
|
||||
intentId: completion.intentId,
|
||||
deliveries: [{ id: deliveryId, state: "prepared" }],
|
||||
},
|
||||
},
|
||||
);
|
||||
const sendMatrix = vi.fn().mockResolvedValue({ messageId: "matrix-message-1" });
|
||||
const params = {
|
||||
cfg: {} as OpenClawConfig,
|
||||
channel: "matrix" as const,
|
||||
to: "!room:example",
|
||||
payloads: [{ text: "deliver once" }],
|
||||
deps: { matrix: sendMatrix },
|
||||
queuePolicy: "required" as const,
|
||||
deliveryIntentId: deliveryId,
|
||||
deliveryCompletion: completion,
|
||||
};
|
||||
|
||||
await expect(deliverOutboundPayloads(params)).resolves.toMatchObject([
|
||||
{ messageId: "matrix-message-1" },
|
||||
]);
|
||||
expect(loadSessionEntry({ sessionKey, storePath })?.pendingFinalDelivery?.deliveries).toEqual([
|
||||
{ id: deliveryId, state: "delivered" },
|
||||
]);
|
||||
|
||||
await expect(deliverOutboundPayloads(params)).resolves.toEqual([]);
|
||||
expect(sendMatrix).toHaveBeenCalledOnce();
|
||||
expect(await loadPendingDeliveries(tmpDir)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { commitMainSessionRecovery } from "../../agents/main-session-recovery/main-session-recovery-store.js";
|
||||
import { loadSessionEntry, replaceSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import type { InternalSessionEntry } from "../../config/sessions/types.js";
|
||||
import { settlePendingFinalDelivery } from "./delivery-completion.js";
|
||||
|
||||
const recoveryMocks = vi.hoisted(() => ({
|
||||
scheduleMainSessionRecoveryPendingTarget: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock(
|
||||
"../../agents/main-session-recovery/main-session-recovery-owner-release.js",
|
||||
() => recoveryMocks,
|
||||
);
|
||||
|
||||
describe("pending-final delivery completion", () => {
|
||||
let tmpDir: string;
|
||||
let storePath: string;
|
||||
const sessionKey = "agent:main:main";
|
||||
const completion = {
|
||||
kind: "pending-final" as const,
|
||||
deliveryId: "delivery-1",
|
||||
intentId: "intent-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
storePath: "",
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-delivery-completion-"));
|
||||
storePath = path.join(tmpDir, "sessions.json");
|
||||
completion.storePath = storePath;
|
||||
const entry: InternalSessionEntry = {
|
||||
sessionId: completion.sessionId,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
updatedAt: Date.now(),
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
pendingFinalDelivery: {
|
||||
kind: "replayable",
|
||||
text: "durable final",
|
||||
createdAt: Date.now(),
|
||||
intentId: completion.intentId,
|
||||
deliveries: [{ id: completion.deliveryId, state: "prepared" }],
|
||||
},
|
||||
};
|
||||
await replaceSessionEntry({ sessionKey, storePath }, entry);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("invalidates an earlier recovery decision and wakes the exact session", async () => {
|
||||
const observation = { sessionId: completion.sessionId, cycleId: "cycle-1", revision: 1 };
|
||||
|
||||
await expect(settlePendingFinalDelivery(completion, "delivered")).resolves.toEqual({
|
||||
state: "delivered",
|
||||
});
|
||||
|
||||
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
|
||||
mainRestartRecovery: { revision: 2 },
|
||||
pendingFinalDelivery: {
|
||||
deliveries: [{ id: completion.deliveryId, state: "delivered" }],
|
||||
},
|
||||
});
|
||||
expect(recoveryMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith({
|
||||
sessionId: completion.sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
await expect(
|
||||
commitMainSessionRecovery({
|
||||
command: { kind: "fail_recovery", now: Date.now(), observation },
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey, storePath },
|
||||
}),
|
||||
).resolves.toMatchObject({ transition: { kind: "rejected", reason: "stale_revision" } });
|
||||
});
|
||||
|
||||
it("records queue custody without waking recovery", async () => {
|
||||
await expect(settlePendingFinalDelivery(completion, "queued")).resolves.toEqual({
|
||||
state: "queued",
|
||||
});
|
||||
|
||||
expect(loadSessionEntry({ sessionKey, storePath })).toMatchObject({
|
||||
mainRestartRecovery: { revision: 2 },
|
||||
pendingFinalDelivery: {
|
||||
deliveries: [{ id: completion.deliveryId, state: "queued" }],
|
||||
},
|
||||
});
|
||||
expect(recoveryMocks.scheduleMainSessionRecoveryPendingTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("carries the custom queue root when a terminal sibling wakes recovery", async () => {
|
||||
const entry = loadSessionEntry({ sessionKey, storePath })!;
|
||||
await replaceSessionEntry(
|
||||
{ sessionKey, storePath },
|
||||
{
|
||||
...entry,
|
||||
pendingFinalDelivery: {
|
||||
...entry.pendingFinalDelivery!,
|
||||
deliveries: [
|
||||
{ id: completion.deliveryId, state: "prepared" },
|
||||
{ id: "delivery-2", state: "queued" },
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await expect(
|
||||
settlePendingFinalDelivery(completion, "delivered", undefined, tmpDir),
|
||||
).resolves.toEqual({ state: "delivered" });
|
||||
|
||||
expect(recoveryMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith({
|
||||
sessionId: completion.sessionId,
|
||||
sessionKey,
|
||||
stateDir: tmpDir,
|
||||
storePath,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -7,74 +7,213 @@ import {
|
||||
markConversationDeliveryUnknown,
|
||||
type ConversationDeliveryRecord,
|
||||
} from "../../config/sessions/conversation-delivery-store.js";
|
||||
import { updateSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import type { InternalSessionEntry } from "../../config/sessions/types.js";
|
||||
import type { OutboundDeliveryResult } from "./deliver-types.js";
|
||||
|
||||
/** Serializable owner callback for a durable queue entry. */
|
||||
export type DurableDeliveryCompletion = {
|
||||
kind: "conversation";
|
||||
agentId: string;
|
||||
operationId: string;
|
||||
storePath?: string;
|
||||
export type DurableDeliveryCompletion =
|
||||
| {
|
||||
kind: "conversation";
|
||||
agentId: string;
|
||||
operationId: string;
|
||||
storePath?: string;
|
||||
}
|
||||
| {
|
||||
kind: "pending-final";
|
||||
deliveryId: string;
|
||||
intentId: string;
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
};
|
||||
|
||||
type DurableDeliveryCompletionResult = {
|
||||
state: "prepared" | "queued" | "delivered" | "suppressed" | "rejected" | "unknown" | "stale";
|
||||
platformMessageId?: string;
|
||||
rejectionError?: string;
|
||||
};
|
||||
|
||||
function scopeForCompletion(completion: DurableDeliveryCompletion) {
|
||||
function scopeForCompletion(
|
||||
completion: Extract<DurableDeliveryCompletion, { kind: "conversation" }>,
|
||||
) {
|
||||
return {
|
||||
agentId: completion.agentId,
|
||||
...(completion.storePath ? { storePath: completion.storePath } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function conversationResult(record: ConversationDeliveryRecord): DurableDeliveryCompletionResult {
|
||||
const delivered = record.status === "sent" || record.status === "replied";
|
||||
return {
|
||||
state: delivered
|
||||
? "delivered"
|
||||
: record.status === "suppressed" ||
|
||||
record.status === "rejected" ||
|
||||
record.status === "unknown"
|
||||
? record.status
|
||||
: "queued",
|
||||
...(delivered && (record.platformMessageId || record.preparedMessageId)
|
||||
? { platformMessageId: record.platformMessageId ?? record.preparedMessageId }
|
||||
: {}),
|
||||
...(record.status === "rejected" && record.rejectionError
|
||||
? { rejectionError: record.rejectionError }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function settlePendingFinalDelivery(
|
||||
completion: Extract<DurableDeliveryCompletion, { kind: "pending-final" }>,
|
||||
state: Exclude<DurableDeliveryCompletionResult["state"], "rejected" | "stale">,
|
||||
expectedStates?: readonly ("prepared" | "queued" | "unknown")[],
|
||||
stateDir?: string,
|
||||
): Promise<DurableDeliveryCompletionResult> {
|
||||
let settled: DurableDeliveryCompletionResult["state"] = "stale";
|
||||
let wakeRecovery = false;
|
||||
await updateSessionEntry(
|
||||
{ sessionKey: completion.sessionKey, storePath: completion.storePath },
|
||||
(entry) => {
|
||||
const internalEntry: InternalSessionEntry = entry;
|
||||
if (
|
||||
internalEntry.sessionId !== completion.sessionId ||
|
||||
internalEntry.pendingFinalDelivery?.intentId !== completion.intentId
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const deliveries = internalEntry.pendingFinalDelivery.deliveries;
|
||||
const index = deliveries?.findIndex(({ id }) => id === completion.deliveryId) ?? -1;
|
||||
if (!deliveries || index < 0) {
|
||||
return null;
|
||||
}
|
||||
const current = deliveries[index]!.state;
|
||||
if (expectedStates && !expectedStates.some((expected) => expected === current)) {
|
||||
return null;
|
||||
}
|
||||
const terminal =
|
||||
current === "delivered" ||
|
||||
current === "suppressed" ||
|
||||
(current === "unknown" && state === "unknown");
|
||||
settled = terminal ? current : state;
|
||||
if (settled === current) {
|
||||
return null;
|
||||
}
|
||||
wakeRecovery =
|
||||
settled !== "queued" &&
|
||||
internalEntry.status === "running" &&
|
||||
internalEntry.abortedLastRun === true;
|
||||
return {
|
||||
...(internalEntry.mainRestartRecovery
|
||||
? {
|
||||
mainRestartRecovery: {
|
||||
...internalEntry.mainRestartRecovery,
|
||||
revision: internalEntry.mainRestartRecovery.revision + 1,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
pendingFinalDelivery: {
|
||||
...internalEntry.pendingFinalDelivery,
|
||||
deliveries: deliveries.with(index, { id: completion.deliveryId, state: settled }),
|
||||
},
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
{ skipMaintenance: true, takeCacheOwnership: true },
|
||||
);
|
||||
if (wakeRecovery) {
|
||||
const { scheduleMainSessionRecoveryPendingTarget } =
|
||||
await import("../../agents/main-session-recovery/main-session-recovery-owner-release.js");
|
||||
scheduleMainSessionRecoveryPendingTarget({
|
||||
sessionId: completion.sessionId,
|
||||
sessionKey: completion.sessionKey,
|
||||
...(stateDir !== undefined ? { stateDir } : {}),
|
||||
storePath: completion.storePath,
|
||||
});
|
||||
}
|
||||
return { state: settled };
|
||||
}
|
||||
|
||||
function readPlatformMessageId(result: OutboundDeliveryResult): string | undefined {
|
||||
const receiptId = result.receipt ? resolveMessageReceiptPrimaryId(result.receipt) : undefined;
|
||||
return receiptId ?? (result.messageId.trim() || undefined);
|
||||
}
|
||||
|
||||
/** Records queue ownership before either the live sender or recovery crosses platform I/O. */
|
||||
export function markDurableDeliveryQueued(
|
||||
export async function markDurableDeliveryQueued(
|
||||
completion: DurableDeliveryCompletion,
|
||||
queueId: string,
|
||||
): ConversationDeliveryRecord {
|
||||
return markConversationDeliveryQueued(
|
||||
scopeForCompletion(completion),
|
||||
completion.operationId,
|
||||
queueId,
|
||||
);
|
||||
expectedPendingFinalState?: "prepared",
|
||||
): Promise<DurableDeliveryCompletionResult> {
|
||||
return completion.kind === "pending-final"
|
||||
? // The reply dispatcher may have claimed direct custody ("queued") before the
|
||||
// durable enqueue; both states still belong to this send attempt.
|
||||
await settlePendingFinalDelivery(
|
||||
completion,
|
||||
"queued",
|
||||
expectedPendingFinalState ? ["prepared", "queued"] : undefined,
|
||||
)
|
||||
: conversationResult(
|
||||
markConversationDeliveryQueued(
|
||||
scopeForCompletion(completion),
|
||||
completion.operationId,
|
||||
queueId,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Finalizes owner state from identified platform evidence before queue acknowledgement. */
|
||||
export function completeDurableDelivery(
|
||||
export async function completeDurableDelivery(
|
||||
completion: DurableDeliveryCompletion,
|
||||
result: OutboundDeliveryResult,
|
||||
): ConversationDeliveryRecord {
|
||||
return markConversationDeliverySent(
|
||||
scopeForCompletion(completion),
|
||||
completion.operationId,
|
||||
readPlatformMessageId(result),
|
||||
);
|
||||
stateDir?: string,
|
||||
): Promise<DurableDeliveryCompletionResult> {
|
||||
return completion.kind === "pending-final"
|
||||
? await settlePendingFinalDelivery(completion, "delivered", undefined, stateDir)
|
||||
: conversationResult(
|
||||
markConversationDeliverySent(
|
||||
scopeForCompletion(completion),
|
||||
completion.operationId,
|
||||
readPlatformMessageId(result),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Finalizes a policy-suppressed send before its durable intent is acknowledged. */
|
||||
export function suppressDurableDelivery(
|
||||
export async function suppressDurableDelivery(
|
||||
completion: DurableDeliveryCompletion,
|
||||
): ConversationDeliveryRecord {
|
||||
return markConversationDeliverySuppressed(scopeForCompletion(completion), completion.operationId);
|
||||
stateDir?: string,
|
||||
): Promise<DurableDeliveryCompletionResult> {
|
||||
return completion.kind === "pending-final"
|
||||
? await settlePendingFinalDelivery(completion, "suppressed", undefined, stateDir)
|
||||
: conversationResult(
|
||||
markConversationDeliverySuppressed(scopeForCompletion(completion), completion.operationId),
|
||||
);
|
||||
}
|
||||
|
||||
/** Finalizes a permanent provider rejection that provably preceded platform I/O. */
|
||||
export function rejectDurableDelivery(
|
||||
export async function rejectDurableDelivery(
|
||||
completion: DurableDeliveryCompletion,
|
||||
error: string,
|
||||
): ConversationDeliveryRecord {
|
||||
return markConversationDeliveryRejected(
|
||||
scopeForCompletion(completion),
|
||||
completion.operationId,
|
||||
error,
|
||||
);
|
||||
stateDir?: string,
|
||||
): Promise<DurableDeliveryCompletionResult> {
|
||||
return completion.kind === "pending-final"
|
||||
? await settlePendingFinalDelivery(completion, "unknown", undefined, stateDir)
|
||||
: conversationResult(
|
||||
markConversationDeliveryRejected(
|
||||
scopeForCompletion(completion),
|
||||
completion.operationId,
|
||||
error,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Makes a dead-lettered durable send terminal without allowing a blind replay. */
|
||||
export function failDurableDelivery(
|
||||
export async function failDurableDelivery(
|
||||
completion: DurableDeliveryCompletion,
|
||||
): ConversationDeliveryRecord {
|
||||
return markConversationDeliveryUnknown(scopeForCompletion(completion), completion.operationId);
|
||||
stateDir?: string,
|
||||
): Promise<DurableDeliveryCompletionResult> {
|
||||
return completion.kind === "pending-final"
|
||||
? await settlePendingFinalDelivery(completion, "unknown", undefined, stateDir)
|
||||
: conversationResult(
|
||||
markConversationDeliveryUnknown(scopeForCompletion(completion), completion.operationId),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -329,6 +329,7 @@ describe("outbound prepared queue migration", () => {
|
||||
expect(hookMocks.runMessageSending).not.toHaveBeenCalled();
|
||||
expect(completionMocks.failDurableDelivery).toHaveBeenCalledWith(
|
||||
interrupted.deliveryCompletion,
|
||||
tmpDir(),
|
||||
);
|
||||
expect(getDeliveryQueueEntryStatus(OUTBOUND_LEGACY_PREPARATION_QUEUE_NAME, id, tmpDir())).toBe(
|
||||
"failed",
|
||||
|
||||
@@ -320,7 +320,7 @@ async function failInterruptedLegacyPreparation(params: {
|
||||
}
|
||||
if (params.entry.deliveryCompletion) {
|
||||
try {
|
||||
failDurableDelivery(params.entry.deliveryCompletion);
|
||||
await failDurableDelivery(params.entry.deliveryCompletion, params.stateDir);
|
||||
} catch (error) {
|
||||
params.log.warn(
|
||||
`Legacy delivery ${params.entry.id} interrupted preparation owner could not be marked unknown: ${String(error)}`,
|
||||
|
||||
@@ -355,7 +355,7 @@ async function applyRecoveryDeliveryAdmission(params: {
|
||||
if (admission.status === "allowed") {
|
||||
return "allowed";
|
||||
}
|
||||
markDurableDeliveryFailedBestEffort(params.entry, params.log);
|
||||
await markDurableDeliveryFailedBestEffort(params.entry, params.log, params.stateDir);
|
||||
const result = await failPendingDelivery(
|
||||
{
|
||||
id: params.entry.id,
|
||||
@@ -547,7 +547,7 @@ async function moveEntryToFailedWithLogging(
|
||||
log: RecoveryLogger,
|
||||
stateDir?: string,
|
||||
): Promise<boolean> {
|
||||
markDurableDeliveryFailedBestEffort(entry, log);
|
||||
await markDurableDeliveryFailedBestEffort(entry, log, stateDir);
|
||||
try {
|
||||
const attemptId = recoveryPlatformAttemptId(entry);
|
||||
await moveEntryToFailedAndCleanup({ entry, cfg, log, stateDir, attemptId });
|
||||
@@ -599,12 +599,16 @@ async function recordRecoveredFailure(
|
||||
}).fail(record, error);
|
||||
}
|
||||
|
||||
function markDurableDeliveryFailedBestEffort(entry: QueuedDelivery, log: RecoveryLogger): void {
|
||||
async function markDurableDeliveryFailedBestEffort(
|
||||
entry: QueuedDelivery,
|
||||
log: RecoveryLogger,
|
||||
stateDir?: string,
|
||||
): Promise<void> {
|
||||
if (!entry.deliveryCompletion) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
failDurableDelivery(entry.deliveryCompletion);
|
||||
await failDurableDelivery(entry.deliveryCompletion, stateDir);
|
||||
} catch (error) {
|
||||
// Queue ownership is authoritative for replay safety. Missing owner state
|
||||
// must not leave a dead-lettered delivery permanently replayable.
|
||||
@@ -626,9 +630,9 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
|
||||
if (!completion) {
|
||||
return "continue";
|
||||
}
|
||||
let operation: ReturnType<typeof markDurableDeliveryQueued>;
|
||||
let operation: Awaited<ReturnType<typeof markDurableDeliveryQueued>>;
|
||||
try {
|
||||
operation = markDurableDeliveryQueued(completion, opts.entry.id);
|
||||
operation = await markDurableDeliveryQueued(completion, opts.entry.id);
|
||||
} catch (error) {
|
||||
const errMsg = `delivery owner state unavailable: ${formatErrorMessage(error)}`;
|
||||
await recordRecoveredFailure(failDelivery, opts.entry, errMsg, opts.stateDir).catch(
|
||||
@@ -638,7 +642,7 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
|
||||
opts.log.warn(`Delivery entry ${opts.entry.id} ${errMsg}`);
|
||||
return "failed";
|
||||
}
|
||||
if (operation.status === "sent" || operation.status === "replied") {
|
||||
if (operation.state === "delivered") {
|
||||
try {
|
||||
await ackRecoveredDelivery(opts.entry, opts.stateDir);
|
||||
} catch (error) {
|
||||
@@ -647,7 +651,7 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
|
||||
opts.log.warn(`Delivery entry ${opts.entry.id} ${errMsg}`);
|
||||
return "failed";
|
||||
}
|
||||
const messageId = operation.platformMessageId ?? operation.preparedMessageId;
|
||||
const messageId = operation.platformMessageId;
|
||||
if (messageId) {
|
||||
const result: OutboundDeliveryResult = { channel: opts.entry.channel, messageId };
|
||||
emitRecoveredTerminalSuccess(opts.entry, result);
|
||||
@@ -663,7 +667,7 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
|
||||
opts.onRecovered?.(opts.entry);
|
||||
return "recovered";
|
||||
}
|
||||
if (operation.status === "suppressed") {
|
||||
if (operation.state === "suppressed" || operation.state === "stale") {
|
||||
try {
|
||||
await (typeof opts.entry.completionRetention === "object"
|
||||
? ackRecoveredDelivery(opts.entry, opts.stateDir, { suppressCompletionReceipt: true })
|
||||
@@ -677,7 +681,7 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
|
||||
opts.onRecovered?.(opts.entry);
|
||||
return "recovered";
|
||||
}
|
||||
if (operation.status === "rejected") {
|
||||
if (operation.state === "rejected") {
|
||||
try {
|
||||
await (typeof opts.entry.completionRetention === "object"
|
||||
? ackRecoveredDelivery(opts.entry, opts.stateDir, { suppressCompletionReceipt: true })
|
||||
@@ -696,17 +700,13 @@ async function resolveCompletedOwnerBeforeRecovery(opts: {
|
||||
failureStage: "platform_send",
|
||||
}),
|
||||
);
|
||||
emitRecoveredTerminalFailure(
|
||||
opts.entry,
|
||||
operation.rejectionError ?? "delivery permanently rejected before platform dispatch",
|
||||
);
|
||||
opts.onFailed?.(
|
||||
opts.entry,
|
||||
operation.rejectionError ?? "delivery permanently rejected before platform dispatch",
|
||||
);
|
||||
const error =
|
||||
operation.rejectionError ?? "delivery permanently rejected before platform dispatch";
|
||||
emitRecoveredTerminalFailure(opts.entry, error);
|
||||
opts.onFailed?.(opts.entry, error);
|
||||
return "failed";
|
||||
}
|
||||
if (operation.status === "unknown") {
|
||||
if (operation.state === "unknown") {
|
||||
const moved = await moveEntryToFailedWithLogging(opts.entry, opts.cfg, opts.log, opts.stateDir);
|
||||
return moved ? "moved-to-failed" : "failed";
|
||||
}
|
||||
@@ -773,7 +773,7 @@ async function drainQueuedEntry(opts: {
|
||||
try {
|
||||
const result = buildReconciledSentResult(entry, reconciliation);
|
||||
if (entry.deliveryCompletion) {
|
||||
completeDurableDelivery(entry.deliveryCompletion, result);
|
||||
await completeDurableDelivery(entry.deliveryCompletion, result, opts.stateDir);
|
||||
}
|
||||
await ackRecoveredDelivery(entry, opts.stateDir, undefined, entry.platformSendAttemptId);
|
||||
emitRecoveredTerminalSuccess(entry, result);
|
||||
@@ -850,7 +850,7 @@ async function drainQueuedEntry(opts: {
|
||||
return "failed";
|
||||
}
|
||||
try {
|
||||
markDurableDeliveryFailedBestEffort(entry, opts.log);
|
||||
await markDurableDeliveryFailedBestEffort(entry, opts.log, opts.stateDir);
|
||||
const attemptId = recoveryPlatformAttemptId(entry);
|
||||
await moveEntryToFailedAndCleanup({
|
||||
entry,
|
||||
@@ -927,7 +927,7 @@ async function drainQueuedEntry(opts: {
|
||||
: await reserveDeliveryAttempt(entry.id, maxRetries, opts.stateDir);
|
||||
if (reservation.status === "exhausted") {
|
||||
const errMsg = `delivery retry budget exhausted (${reservation.attemptCount}/${maxRetries})`;
|
||||
markDurableDeliveryFailedBestEffort(entry, opts.log);
|
||||
await markDurableDeliveryFailedBestEffort(entry, opts.log, opts.stateDir);
|
||||
try {
|
||||
await moveEntryToFailedAndCleanup({
|
||||
entry,
|
||||
@@ -993,11 +993,6 @@ async function drainQueuedEntry(opts: {
|
||||
}
|
||||
if (results.length > 0) {
|
||||
deliveredResults = [...results];
|
||||
if (entry.deliveryCompletion) {
|
||||
completeDurableDelivery(entry.deliveryCompletion, results.at(-1)!);
|
||||
}
|
||||
} else if (entry.deliveryCompletion) {
|
||||
suppressDurableDelivery(entry.deliveryCompletion);
|
||||
}
|
||||
const failedOutcomes = payloadOutcomes.filter((outcome) => outcome.status === "failed");
|
||||
const failedOutcome = failedOutcomes[0];
|
||||
@@ -1035,6 +1030,13 @@ async function drainQueuedEntry(opts: {
|
||||
}
|
||||
return "failed";
|
||||
}
|
||||
if (entry.deliveryCompletion) {
|
||||
if (results.length > 0) {
|
||||
await completeDurableDelivery(entry.deliveryCompletion, results.at(-1)!, opts.stateDir);
|
||||
} else {
|
||||
await suppressDurableDelivery(entry.deliveryCompletion, opts.stateDir);
|
||||
}
|
||||
}
|
||||
postSendState ??=
|
||||
results.length > 0
|
||||
? await persistRecoveredPostSendState({
|
||||
@@ -1155,9 +1157,13 @@ async function drainQueuedEntry(opts: {
|
||||
if (permanentPlatformRejection || isPermanentDeliveryError(errMsg)) {
|
||||
try {
|
||||
if (permanentPlatformRejection && entry.deliveryCompletion) {
|
||||
rejectDurableDelivery(entry.deliveryCompletion, permanentPlatformRejection.message);
|
||||
await rejectDurableDelivery(
|
||||
entry.deliveryCompletion,
|
||||
permanentPlatformRejection.message,
|
||||
opts.stateDir,
|
||||
);
|
||||
} else {
|
||||
markDurableDeliveryFailedBestEffort(entry, opts.log);
|
||||
await markDurableDeliveryFailedBestEffort(entry, opts.log, opts.stateDir);
|
||||
}
|
||||
await moveEntryToFailedAndCleanup({
|
||||
entry,
|
||||
@@ -1256,7 +1262,7 @@ export async function drainPendingDeliveries(opts: {
|
||||
!needsUnknownSendReconciliation(currentEntry)
|
||||
) {
|
||||
try {
|
||||
markDurableDeliveryFailedBestEffort(currentEntry, opts.log);
|
||||
await markDurableDeliveryFailedBestEffort(currentEntry, opts.log, opts.stateDir);
|
||||
const attemptId = recoveryPlatformAttemptId(currentEntry);
|
||||
await moveEntryToFailedAndCleanup({
|
||||
entry: currentEntry,
|
||||
|
||||
Reference in New Issue
Block a user