diff --git a/src/gateway/server-methods/chat.ts b/src/gateway/server-methods/chat.ts index 52b1dd086b3c..e6a203e10275 100644 --- a/src/gateway/server-methods/chat.ts +++ b/src/gateway/server-methods/chat.ts @@ -21,6 +21,7 @@ import { stageSandboxMedia } from "../../auto-reply/reply/stage-sandbox-media.js import type { MsgContext, TemplateContext } from "../../auto-reply/templating.js"; import { extractCanvasFromText } from "../../chat/canvas-render.js"; import { resolveSessionFilePath } from "../../config/sessions.js"; +import { resolveMirroredTranscriptText } from "../../config/sessions/transcript-mirror.js"; import { streamSessionTranscriptLines } from "../../config/sessions/transcript-stream.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { @@ -607,6 +608,18 @@ function hasAssistantDisplayMediaContent( return Boolean(content?.some((block) => block?.type !== "text")); } +function hasManagedOutgoingAssistantContent( + content: readonly AssistantDisplayContentBlock[] | undefined, +): boolean { + return Boolean( + content?.some( + (block) => + block?.type === "image" && + (isManagedOutgoingImageUrl(block.url) || isManagedOutgoingImageUrl(block.openUrl)), + ), + ); +} + function scheduleChatHistoryManagedImageCleanup(params: { sessionKey: string; context: Pick; @@ -1439,6 +1452,103 @@ async function transcriptHasIdempotencyKey( } } +async function findAssistantTranscriptMessageByIdempotencyKey( + transcriptPath: string, + idempotencyKey: string, +): Promise<{ messageId: string; message: Record } | null> { + const trimmedIdempotencyKey = idempotencyKey.trim(); + if (!trimmedIdempotencyKey) { + return null; + } + const index = await readSessionTranscriptIndex(transcriptPath); + const target = index?.entries.toReversed().find((entry) => { + const message = entry.record.message as Record | undefined; + return ( + typeof entry.id === "string" && + entry.id.trim().length > 0 && + message?.role === "assistant" && + message.idempotencyKey === trimmedIdempotencyKey + ); + }); + const message = target?.record.message as Record | undefined; + if (!target?.id || !message) { + return null; + } + return { messageId: target.id, message }; +} + +async function findSourceReplyTranscriptMirrorByIdempotencyKey( + transcriptPath: string, + idempotencyKey: string, +): Promise<{ messageId: string; message: Record } | null> { + const found = await findAssistantTranscriptMessageByIdempotencyKey( + transcriptPath, + idempotencyKey, + ); + if (found?.message.provider !== "openclaw" || found.message.model !== "delivery-mirror") { + return null; + } + return found; +} + +function extractAssistantTranscriptText(message: Record): string | undefined { + const content = message.content; + if (!Array.isArray(content)) { + return undefined; + } + const text = content + .map((block) => + block && + typeof block === "object" && + (block as { type?: unknown }).type === "text" && + typeof (block as { text?: unknown }).text === "string" + ? ((block as { text: string }).text.trim() ?? "") + : "", + ) + .filter(Boolean) + .join("\n") + .trim(); + return text || undefined; +} + +async function findSourceReplyTranscriptMirrorByMetadata(params: { + transcriptPath: string; + idempotencyKey: string; + metadata: NonNullable>["sourceReplyTranscriptMirror"]; +}): Promise<{ messageId: string; message: Record } | null> { + const byIdempotencyKey = await findSourceReplyTranscriptMirrorByIdempotencyKey( + params.transcriptPath, + params.idempotencyKey, + ); + if (byIdempotencyKey) { + return byIdempotencyKey; + } + const expectedText = resolveMirroredTranscriptText({ + text: params.metadata?.text, + mediaUrls: params.metadata?.mediaUrls, + }); + if (!expectedText) { + return null; + } + const index = await readSessionTranscriptIndex(params.transcriptPath); + const target = index?.entries.toReversed().find((entry) => { + const message = entry.record.message as Record | undefined; + return ( + typeof entry.id === "string" && + entry.id.trim().length > 0 && + message?.role === "assistant" && + message.provider === "openclaw" && + message.model === "delivery-mirror" && + extractAssistantTranscriptText(message) === expectedText + ); + }); + const message = target?.record.message as Record | undefined; + if (!target?.id || !message) { + return null; + } + return { messageId: target.id, message }; +} + async function appendAssistantTranscriptMessage(params: { message: string; label?: string; @@ -1484,7 +1594,13 @@ async function appendAssistantTranscriptMessage(params: { params.idempotencyKey && (await transcriptHasIdempotencyKey(transcriptPath, params.idempotencyKey)) ) { - return { ok: true }; + const existing = await findAssistantTranscriptMessageByIdempotencyKey( + transcriptPath, + params.idempotencyKey, + ); + return existing + ? { ok: true, messageId: existing.messageId, message: existing.message } + : { ok: true }; } return await appendInjectedAssistantMessageToTranscript({ @@ -3101,50 +3217,269 @@ export const chatHandlers: GatewayRequestHandlers = { getAgentScopedMediaLocalRoots(cfg, agentId), resolvedTranscriptPath ? [resolvedTranscriptPath] : undefined, ); - const assistantContent = await buildAssistantDisplayContentFromReplyPayloads({ - sessionKey, - payloads: finalPayloads, - managedImageLocalRoots: mediaLocalRoots, - includeSensitiveMedia: false, - onLocalAudioAccessDenied: (message) => { - context.logGateway.warn( - `webchat audio embedding denied local path: ${message}`, - ); - }, - onManagedImagePrepareError: (message) => { - context.logGateway.warn( - `webchat image embedding skipped attachment: ${message}`, - ); - }, - }); - const mediaMessage = await buildWebchatAssistantMediaMessage(finalPayloads, { - localRoots: mediaLocalRoots, - onLocalAudioAccessDenied: (message) => { - context.logGateway.warn( - `webchat audio embedding denied local path: ${message}`, - ); - }, - }); - const broadcastAssistantContent = hasAssistantDisplayMediaContent( - assistantContent, - ) - ? assistantContent - : hasAssistantDisplayMediaContent(mediaMessage?.content) - ? mediaMessage?.content - : assistantContent; + const buildReplyAssistantContent = async ( + payloads: typeof finalPayloads, + ): Promise => + await buildAssistantDisplayContentFromReplyPayloads({ + sessionKey, + payloads, + managedImageLocalRoots: mediaLocalRoots, + includeSensitiveMedia: false, + onLocalAudioAccessDenied: (message) => { + context.logGateway.warn( + `webchat audio embedding denied local path: ${message}`, + ); + }, + onManagedImagePrepareError: (message) => { + context.logGateway.warn( + `webchat image embedding skipped attachment: ${message}`, + ); + }, + }); + const buildReplyMediaMessage = async (payloads: typeof finalPayloads) => + await buildWebchatAssistantMediaMessage(payloads, { + localRoots: mediaLocalRoots, + onLocalAudioAccessDenied: (message) => { + context.logGateway.warn( + `webchat audio embedding denied local path: ${message}`, + ); + }, + }); + const combinedAssistantContent = + sourceReplyPayloads.length === 1 + ? await buildReplyAssistantContent(finalPayloads) + : undefined; + const combinedMediaMessage = + sourceReplyPayloads.length === 1 + ? await buildReplyMediaMessage(finalPayloads) + : undefined; + type SourceReplyContentState = { + broadcastContent: AssistantDisplayContentBlock[]; + persistedContent: AssistantDisplayContentBlock[]; + hasManagedOutgoingContent: boolean; + backedManagedOutgoingContent: boolean; + }; + const sourceReplyContentStates: SourceReplyContentState[] = []; + const sourceReplyBroadcastContent: AssistantDisplayContentBlock[] = []; + for (const [replyIndex] of sourceReplyPayloads.entries()) { + const finalPayload = finalPayloads[replyIndex]; + if (!finalPayload) { + continue; + } + const replyAssistantContent = + sourceReplyPayloads.length === 1 + ? combinedAssistantContent + : await buildReplyAssistantContent([finalPayload]); + const replyMediaMessage = + sourceReplyPayloads.length === 1 + ? combinedMediaMessage + : await buildReplyMediaMessage([finalPayload]); + const replyBroadcastContent = hasAssistantDisplayMediaContent( + replyAssistantContent, + ) + ? replyAssistantContent + : hasAssistantDisplayMediaContent(replyMediaMessage?.content) + ? replyMediaMessage?.content + : replyAssistantContent; + const persistedContent = replaceAssistantContentTextBlocks( + replyAssistantContent, + replyMediaMessage ?? null, + ); + const state: SourceReplyContentState = { + broadcastContent: replyBroadcastContent ? [...replyBroadcastContent] : [], + persistedContent: persistedContent ? [...persistedContent] : [], + hasManagedOutgoingContent: + hasManagedOutgoingAssistantContent(persistedContent), + backedManagedOutgoingContent: false, + }; + sourceReplyContentStates[replyIndex] = state; + if (state.broadcastContent.length > 0) { + sourceReplyBroadcastContent.push(...state.broadcastContent); + } + } + const displayReply = - extractAssistantDisplayTextFromContent(assistantContent) ?? + extractAssistantDisplayTextFromContent(sourceReplyBroadcastContent) ?? buildTranscriptReplyText(finalPayloads); - if (broadcastAssistantContent?.length || displayReply) { + if (sourceReplyBroadcastContent.length || displayReply) { + const sourceReplyPersistenceRequests: Array<{ + idempotencyKey: string; + metadata: NonNullable< + ReturnType + >["sourceReplyTranscriptMirror"]; + state: SourceReplyContentState; + }> = []; + for (const [replyIndex, sourceReplyPayload] of sourceReplyPayloads.entries()) { + const state = sourceReplyContentStates[replyIndex]; + if (!state || !hasAssistantDisplayMediaContent(state.persistedContent)) { + continue; + } + const mirrorMetadata = + getReplyPayloadMetadata(sourceReplyPayload)?.sourceReplyTranscriptMirror; + const mirrorIdempotencyKey = mirrorMetadata?.idempotencyKey; + if ( + typeof mirrorIdempotencyKey !== "string" || + mirrorIdempotencyKey.trim().length === 0 + ) { + continue; + } + if (!state.hasManagedOutgoingContent) { + state.backedManagedOutgoingContent = true; + } + sourceReplyPersistenceRequests.push({ + idempotencyKey: mirrorIdempotencyKey, + metadata: mirrorMetadata, + state, + }); + } + + const attachSourceReplyManagedImages = async (params: { + messageId?: string; + request: (typeof sourceReplyPersistenceRequests)[number]; + }) => { + if (!params.request.state.hasManagedOutgoingContent) { + params.request.state.backedManagedOutgoingContent = true; + return; + } + if (!params.messageId) { + return; + } + await attachManagedOutgoingImagesToMessage({ + messageId: params.messageId, + blocks: params.request.state.persistedContent, + }); + params.request.state.backedManagedOutgoingContent = true; + }; + + if (resolvedTranscriptPath && sourceReplyPersistenceRequests.length > 0) { + const allowedSourceReplyMirrorIds = new Set(); + for (const [ + replyIndex, + sourceReplyPayload, + ] of sourceReplyPayloads.entries()) { + if (!sourceReplyContentStates[replyIndex]) { + continue; + } + const mirrorIdempotencyKey = + getReplyPayloadMetadata(sourceReplyPayload)?.sourceReplyTranscriptMirror + ?.idempotencyKey; + const mirrorMetadata = + getReplyPayloadMetadata(sourceReplyPayload)?.sourceReplyTranscriptMirror; + if ( + typeof mirrorIdempotencyKey !== "string" || + mirrorIdempotencyKey.trim().length === 0 || + !mirrorMetadata + ) { + continue; + } + const target = await findSourceReplyTranscriptMirrorByMetadata({ + transcriptPath: resolvedTranscriptPath, + idempotencyKey: mirrorIdempotencyKey, + metadata: mirrorMetadata, + }); + if (target) { + allowedSourceReplyMirrorIds.add(target.messageId); + } + } + const rewriteTargets: Array<{ + request: (typeof sourceReplyPersistenceRequests)[number]; + messageId: string; + message: Record; + }> = []; + for (const request of sourceReplyPersistenceRequests) { + const target = await findSourceReplyTranscriptMirrorByMetadata({ + transcriptPath: resolvedTranscriptPath, + idempotencyKey: request.idempotencyKey, + metadata: request.metadata, + }); + if (target) { + rewriteTargets.push({ request, ...target }); + } + } + + if (rewriteTargets.length > 0) { + const rewriteTargetIds = new Set( + rewriteTargets.map((target) => target.messageId), + ); + const rewriteIndex = + await readSessionTranscriptIndex(resolvedTranscriptPath); + const firstRewriteEntryIndex = + rewriteIndex?.entries.findIndex( + (entry) => + typeof entry.id === "string" && rewriteTargetIds.has(entry.id), + ) ?? -1; + const canRewriteSourceReplyMirrors = + firstRewriteEntryIndex >= 0 && + rewriteIndex?.entries + .slice(firstRewriteEntryIndex) + .every( + (entry) => + typeof entry.id !== "string" || + allowedSourceReplyMirrorIds.has(entry.id), + ) === true; + if (canRewriteSourceReplyMirrors) { + const result = await rewriteTranscriptEntriesInSessionFile({ + sessionFile: resolvedTranscriptPath, + sessionKey, + config: cfg, + request: { + allowedRewriteSuffixEntryIds: [...allowedSourceReplyMirrorIds], + replacements: rewriteTargets.map((target) => ({ + entryId: target.messageId, + message: { + ...(target.message as unknown as AgentMessage), + idempotencyKey: target.request.idempotencyKey, + content: target.request.state.persistedContent, + } as unknown as AgentMessage, + })), + }, + }); + if (result.changed) { + for (const target of rewriteTargets) { + const rewritten = + await findSourceReplyTranscriptMirrorByIdempotencyKey( + resolvedTranscriptPath, + target.request.idempotencyKey, + ); + await attachSourceReplyManagedImages({ + messageId: rewritten?.messageId, + request: target.request, + }); + } + } + } + } + } + const sourceReplyContent = sourceReplyContentStates + .flatMap((state) => { + if ( + state.hasManagedOutgoingContent && + !state.backedManagedOutgoingContent + ) { + const stripped = stripManagedOutgoingAssistantContentBlocks( + state.broadcastContent, + ); + return stripped?.length + ? stripped + : [{ type: "text", text: "Media reply could not be displayed." }]; + } + return state.broadcastContent; + }) + .filter((block): block is AssistantDisplayContentBlock => Boolean(block)); + const sourceReplyTextFromContent = + extractAssistantDisplayTextFromContent(sourceReplyContent); + const sourceReplyText = + sourceReplyTextFromContent ?? + (sourceReplyContent.length === 0 ? displayReply : undefined); const now = Date.now(); const message = { role: "assistant", - ...(broadcastAssistantContent?.length - ? { content: broadcastAssistantContent } - : displayReply - ? { content: [{ type: "text", text: displayReply }] } + ...(sourceReplyContent?.length + ? { content: sourceReplyContent } + : sourceReplyText + ? { content: [{ type: "text", text: sourceReplyText }] } : {}), - ...(displayReply ? { text: displayReply } : {}), + ...(sourceReplyText ? { text: sourceReplyText } : {}), timestamp: now, stopReason: "stop", usage: { input: 0, output: 0, totalTokens: 0 },