mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 20:35:39 -06:00
fix: back source reply media in transcripts
This commit is contained in:
Shakker
@@ -21,6 +21,7 @@ import { stageSandboxMedia } from "../../auto-reply/reply/stage-sandbox-media.js
|
||||
import type { MsgContext, TemplateContext } from "../../auto-reply/templating.js";
|
||||
import { extractCanvasFromText } from "../../chat/canvas-render.js";
|
||||
import { resolveSessionFilePath } from "../../config/sessions.js";
|
||||
import { resolveMirroredTranscriptText } from "../../config/sessions/transcript-mirror.js";
|
||||
import { streamSessionTranscriptLines } from "../../config/sessions/transcript-stream.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import {
|
||||
@@ -607,6 +608,18 @@ function hasAssistantDisplayMediaContent(
|
||||
return Boolean(content?.some((block) => block?.type !== "text"));
|
||||
}
|
||||
|
||||
function hasManagedOutgoingAssistantContent(
|
||||
content: readonly AssistantDisplayContentBlock[] | undefined,
|
||||
): boolean {
|
||||
return Boolean(
|
||||
content?.some(
|
||||
(block) =>
|
||||
block?.type === "image" &&
|
||||
(isManagedOutgoingImageUrl(block.url) || isManagedOutgoingImageUrl(block.openUrl)),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function scheduleChatHistoryManagedImageCleanup(params: {
|
||||
sessionKey: string;
|
||||
context: Pick<GatewayRequestContext, "logGateway">;
|
||||
@@ -1439,6 +1452,103 @@ async function transcriptHasIdempotencyKey(
|
||||
}
|
||||
}
|
||||
|
||||
async function findAssistantTranscriptMessageByIdempotencyKey(
|
||||
transcriptPath: string,
|
||||
idempotencyKey: string,
|
||||
): Promise<{ messageId: string; message: Record<string, unknown> } | null> {
|
||||
const trimmedIdempotencyKey = idempotencyKey.trim();
|
||||
if (!trimmedIdempotencyKey) {
|
||||
return null;
|
||||
}
|
||||
const index = await readSessionTranscriptIndex(transcriptPath);
|
||||
const target = index?.entries.toReversed().find((entry) => {
|
||||
const message = entry.record.message as Record<string, unknown> | undefined;
|
||||
return (
|
||||
typeof entry.id === "string" &&
|
||||
entry.id.trim().length > 0 &&
|
||||
message?.role === "assistant" &&
|
||||
message.idempotencyKey === trimmedIdempotencyKey
|
||||
);
|
||||
});
|
||||
const message = target?.record.message as Record<string, unknown> | undefined;
|
||||
if (!target?.id || !message) {
|
||||
return null;
|
||||
}
|
||||
return { messageId: target.id, message };
|
||||
}
|
||||
|
||||
async function findSourceReplyTranscriptMirrorByIdempotencyKey(
|
||||
transcriptPath: string,
|
||||
idempotencyKey: string,
|
||||
): Promise<{ messageId: string; message: Record<string, unknown> } | null> {
|
||||
const found = await findAssistantTranscriptMessageByIdempotencyKey(
|
||||
transcriptPath,
|
||||
idempotencyKey,
|
||||
);
|
||||
if (found?.message.provider !== "openclaw" || found.message.model !== "delivery-mirror") {
|
||||
return null;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
function extractAssistantTranscriptText(message: Record<string, unknown>): string | undefined {
|
||||
const content = message.content;
|
||||
if (!Array.isArray(content)) {
|
||||
return undefined;
|
||||
}
|
||||
const text = content
|
||||
.map((block) =>
|
||||
block &&
|
||||
typeof block === "object" &&
|
||||
(block as { type?: unknown }).type === "text" &&
|
||||
typeof (block as { text?: unknown }).text === "string"
|
||||
? ((block as { text: string }).text.trim() ?? "")
|
||||
: "",
|
||||
)
|
||||
.filter(Boolean)
|
||||
.join("\n")
|
||||
.trim();
|
||||
return text || undefined;
|
||||
}
|
||||
|
||||
async function findSourceReplyTranscriptMirrorByMetadata(params: {
|
||||
transcriptPath: string;
|
||||
idempotencyKey: string;
|
||||
metadata: NonNullable<ReturnType<typeof getReplyPayloadMetadata>>["sourceReplyTranscriptMirror"];
|
||||
}): Promise<{ messageId: string; message: Record<string, unknown> } | null> {
|
||||
const byIdempotencyKey = await findSourceReplyTranscriptMirrorByIdempotencyKey(
|
||||
params.transcriptPath,
|
||||
params.idempotencyKey,
|
||||
);
|
||||
if (byIdempotencyKey) {
|
||||
return byIdempotencyKey;
|
||||
}
|
||||
const expectedText = resolveMirroredTranscriptText({
|
||||
text: params.metadata?.text,
|
||||
mediaUrls: params.metadata?.mediaUrls,
|
||||
});
|
||||
if (!expectedText) {
|
||||
return null;
|
||||
}
|
||||
const index = await readSessionTranscriptIndex(params.transcriptPath);
|
||||
const target = index?.entries.toReversed().find((entry) => {
|
||||
const message = entry.record.message as Record<string, unknown> | undefined;
|
||||
return (
|
||||
typeof entry.id === "string" &&
|
||||
entry.id.trim().length > 0 &&
|
||||
message?.role === "assistant" &&
|
||||
message.provider === "openclaw" &&
|
||||
message.model === "delivery-mirror" &&
|
||||
extractAssistantTranscriptText(message) === expectedText
|
||||
);
|
||||
});
|
||||
const message = target?.record.message as Record<string, unknown> | undefined;
|
||||
if (!target?.id || !message) {
|
||||
return null;
|
||||
}
|
||||
return { messageId: target.id, message };
|
||||
}
|
||||
|
||||
async function appendAssistantTranscriptMessage(params: {
|
||||
message: string;
|
||||
label?: string;
|
||||
@@ -1484,7 +1594,13 @@ async function appendAssistantTranscriptMessage(params: {
|
||||
params.idempotencyKey &&
|
||||
(await transcriptHasIdempotencyKey(transcriptPath, params.idempotencyKey))
|
||||
) {
|
||||
return { ok: true };
|
||||
const existing = await findAssistantTranscriptMessageByIdempotencyKey(
|
||||
transcriptPath,
|
||||
params.idempotencyKey,
|
||||
);
|
||||
return existing
|
||||
? { ok: true, messageId: existing.messageId, message: existing.message }
|
||||
: { ok: true };
|
||||
}
|
||||
|
||||
return await appendInjectedAssistantMessageToTranscript({
|
||||
@@ -3101,50 +3217,269 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
getAgentScopedMediaLocalRoots(cfg, agentId),
|
||||
resolvedTranscriptPath ? [resolvedTranscriptPath] : undefined,
|
||||
);
|
||||
const assistantContent = await buildAssistantDisplayContentFromReplyPayloads({
|
||||
sessionKey,
|
||||
payloads: finalPayloads,
|
||||
managedImageLocalRoots: mediaLocalRoots,
|
||||
includeSensitiveMedia: false,
|
||||
onLocalAudioAccessDenied: (message) => {
|
||||
context.logGateway.warn(
|
||||
`webchat audio embedding denied local path: ${message}`,
|
||||
);
|
||||
},
|
||||
onManagedImagePrepareError: (message) => {
|
||||
context.logGateway.warn(
|
||||
`webchat image embedding skipped attachment: ${message}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
const mediaMessage = await buildWebchatAssistantMediaMessage(finalPayloads, {
|
||||
localRoots: mediaLocalRoots,
|
||||
onLocalAudioAccessDenied: (message) => {
|
||||
context.logGateway.warn(
|
||||
`webchat audio embedding denied local path: ${message}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
const broadcastAssistantContent = hasAssistantDisplayMediaContent(
|
||||
assistantContent,
|
||||
)
|
||||
? assistantContent
|
||||
: hasAssistantDisplayMediaContent(mediaMessage?.content)
|
||||
? mediaMessage?.content
|
||||
: assistantContent;
|
||||
const buildReplyAssistantContent = async (
|
||||
payloads: typeof finalPayloads,
|
||||
): Promise<AssistantDisplayContentBlock[] | undefined> =>
|
||||
await buildAssistantDisplayContentFromReplyPayloads({
|
||||
sessionKey,
|
||||
payloads,
|
||||
managedImageLocalRoots: mediaLocalRoots,
|
||||
includeSensitiveMedia: false,
|
||||
onLocalAudioAccessDenied: (message) => {
|
||||
context.logGateway.warn(
|
||||
`webchat audio embedding denied local path: ${message}`,
|
||||
);
|
||||
},
|
||||
onManagedImagePrepareError: (message) => {
|
||||
context.logGateway.warn(
|
||||
`webchat image embedding skipped attachment: ${message}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
const buildReplyMediaMessage = async (payloads: typeof finalPayloads) =>
|
||||
await buildWebchatAssistantMediaMessage(payloads, {
|
||||
localRoots: mediaLocalRoots,
|
||||
onLocalAudioAccessDenied: (message) => {
|
||||
context.logGateway.warn(
|
||||
`webchat audio embedding denied local path: ${message}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
const combinedAssistantContent =
|
||||
sourceReplyPayloads.length === 1
|
||||
? await buildReplyAssistantContent(finalPayloads)
|
||||
: undefined;
|
||||
const combinedMediaMessage =
|
||||
sourceReplyPayloads.length === 1
|
||||
? await buildReplyMediaMessage(finalPayloads)
|
||||
: undefined;
|
||||
type SourceReplyContentState = {
|
||||
broadcastContent: AssistantDisplayContentBlock[];
|
||||
persistedContent: AssistantDisplayContentBlock[];
|
||||
hasManagedOutgoingContent: boolean;
|
||||
backedManagedOutgoingContent: boolean;
|
||||
};
|
||||
const sourceReplyContentStates: SourceReplyContentState[] = [];
|
||||
const sourceReplyBroadcastContent: AssistantDisplayContentBlock[] = [];
|
||||
for (const [replyIndex] of sourceReplyPayloads.entries()) {
|
||||
const finalPayload = finalPayloads[replyIndex];
|
||||
if (!finalPayload) {
|
||||
continue;
|
||||
}
|
||||
const replyAssistantContent =
|
||||
sourceReplyPayloads.length === 1
|
||||
? combinedAssistantContent
|
||||
: await buildReplyAssistantContent([finalPayload]);
|
||||
const replyMediaMessage =
|
||||
sourceReplyPayloads.length === 1
|
||||
? combinedMediaMessage
|
||||
: await buildReplyMediaMessage([finalPayload]);
|
||||
const replyBroadcastContent = hasAssistantDisplayMediaContent(
|
||||
replyAssistantContent,
|
||||
)
|
||||
? replyAssistantContent
|
||||
: hasAssistantDisplayMediaContent(replyMediaMessage?.content)
|
||||
? replyMediaMessage?.content
|
||||
: replyAssistantContent;
|
||||
const persistedContent = replaceAssistantContentTextBlocks(
|
||||
replyAssistantContent,
|
||||
replyMediaMessage ?? null,
|
||||
);
|
||||
const state: SourceReplyContentState = {
|
||||
broadcastContent: replyBroadcastContent ? [...replyBroadcastContent] : [],
|
||||
persistedContent: persistedContent ? [...persistedContent] : [],
|
||||
hasManagedOutgoingContent:
|
||||
hasManagedOutgoingAssistantContent(persistedContent),
|
||||
backedManagedOutgoingContent: false,
|
||||
};
|
||||
sourceReplyContentStates[replyIndex] = state;
|
||||
if (state.broadcastContent.length > 0) {
|
||||
sourceReplyBroadcastContent.push(...state.broadcastContent);
|
||||
}
|
||||
}
|
||||
|
||||
const displayReply =
|
||||
extractAssistantDisplayTextFromContent(assistantContent) ??
|
||||
extractAssistantDisplayTextFromContent(sourceReplyBroadcastContent) ??
|
||||
buildTranscriptReplyText(finalPayloads);
|
||||
if (broadcastAssistantContent?.length || displayReply) {
|
||||
if (sourceReplyBroadcastContent.length || displayReply) {
|
||||
const sourceReplyPersistenceRequests: Array<{
|
||||
idempotencyKey: string;
|
||||
metadata: NonNullable<
|
||||
ReturnType<typeof getReplyPayloadMetadata>
|
||||
>["sourceReplyTranscriptMirror"];
|
||||
state: SourceReplyContentState;
|
||||
}> = [];
|
||||
for (const [replyIndex, sourceReplyPayload] of sourceReplyPayloads.entries()) {
|
||||
const state = sourceReplyContentStates[replyIndex];
|
||||
if (!state || !hasAssistantDisplayMediaContent(state.persistedContent)) {
|
||||
continue;
|
||||
}
|
||||
const mirrorMetadata =
|
||||
getReplyPayloadMetadata(sourceReplyPayload)?.sourceReplyTranscriptMirror;
|
||||
const mirrorIdempotencyKey = mirrorMetadata?.idempotencyKey;
|
||||
if (
|
||||
typeof mirrorIdempotencyKey !== "string" ||
|
||||
mirrorIdempotencyKey.trim().length === 0
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (!state.hasManagedOutgoingContent) {
|
||||
state.backedManagedOutgoingContent = true;
|
||||
}
|
||||
sourceReplyPersistenceRequests.push({
|
||||
idempotencyKey: mirrorIdempotencyKey,
|
||||
metadata: mirrorMetadata,
|
||||
state,
|
||||
});
|
||||
}
|
||||
|
||||
const attachSourceReplyManagedImages = async (params: {
|
||||
messageId?: string;
|
||||
request: (typeof sourceReplyPersistenceRequests)[number];
|
||||
}) => {
|
||||
if (!params.request.state.hasManagedOutgoingContent) {
|
||||
params.request.state.backedManagedOutgoingContent = true;
|
||||
return;
|
||||
}
|
||||
if (!params.messageId) {
|
||||
return;
|
||||
}
|
||||
await attachManagedOutgoingImagesToMessage({
|
||||
messageId: params.messageId,
|
||||
blocks: params.request.state.persistedContent,
|
||||
});
|
||||
params.request.state.backedManagedOutgoingContent = true;
|
||||
};
|
||||
|
||||
if (resolvedTranscriptPath && sourceReplyPersistenceRequests.length > 0) {
|
||||
const allowedSourceReplyMirrorIds = new Set<string>();
|
||||
for (const [
|
||||
replyIndex,
|
||||
sourceReplyPayload,
|
||||
] of sourceReplyPayloads.entries()) {
|
||||
if (!sourceReplyContentStates[replyIndex]) {
|
||||
continue;
|
||||
}
|
||||
const mirrorIdempotencyKey =
|
||||
getReplyPayloadMetadata(sourceReplyPayload)?.sourceReplyTranscriptMirror
|
||||
?.idempotencyKey;
|
||||
const mirrorMetadata =
|
||||
getReplyPayloadMetadata(sourceReplyPayload)?.sourceReplyTranscriptMirror;
|
||||
if (
|
||||
typeof mirrorIdempotencyKey !== "string" ||
|
||||
mirrorIdempotencyKey.trim().length === 0 ||
|
||||
!mirrorMetadata
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const target = await findSourceReplyTranscriptMirrorByMetadata({
|
||||
transcriptPath: resolvedTranscriptPath,
|
||||
idempotencyKey: mirrorIdempotencyKey,
|
||||
metadata: mirrorMetadata,
|
||||
});
|
||||
if (target) {
|
||||
allowedSourceReplyMirrorIds.add(target.messageId);
|
||||
}
|
||||
}
|
||||
const rewriteTargets: Array<{
|
||||
request: (typeof sourceReplyPersistenceRequests)[number];
|
||||
messageId: string;
|
||||
message: Record<string, unknown>;
|
||||
}> = [];
|
||||
for (const request of sourceReplyPersistenceRequests) {
|
||||
const target = await findSourceReplyTranscriptMirrorByMetadata({
|
||||
transcriptPath: resolvedTranscriptPath,
|
||||
idempotencyKey: request.idempotencyKey,
|
||||
metadata: request.metadata,
|
||||
});
|
||||
if (target) {
|
||||
rewriteTargets.push({ request, ...target });
|
||||
}
|
||||
}
|
||||
|
||||
if (rewriteTargets.length > 0) {
|
||||
const rewriteTargetIds = new Set(
|
||||
rewriteTargets.map((target) => target.messageId),
|
||||
);
|
||||
const rewriteIndex =
|
||||
await readSessionTranscriptIndex(resolvedTranscriptPath);
|
||||
const firstRewriteEntryIndex =
|
||||
rewriteIndex?.entries.findIndex(
|
||||
(entry) =>
|
||||
typeof entry.id === "string" && rewriteTargetIds.has(entry.id),
|
||||
) ?? -1;
|
||||
const canRewriteSourceReplyMirrors =
|
||||
firstRewriteEntryIndex >= 0 &&
|
||||
rewriteIndex?.entries
|
||||
.slice(firstRewriteEntryIndex)
|
||||
.every(
|
||||
(entry) =>
|
||||
typeof entry.id !== "string" ||
|
||||
allowedSourceReplyMirrorIds.has(entry.id),
|
||||
) === true;
|
||||
if (canRewriteSourceReplyMirrors) {
|
||||
const result = await rewriteTranscriptEntriesInSessionFile({
|
||||
sessionFile: resolvedTranscriptPath,
|
||||
sessionKey,
|
||||
config: cfg,
|
||||
request: {
|
||||
allowedRewriteSuffixEntryIds: [...allowedSourceReplyMirrorIds],
|
||||
replacements: rewriteTargets.map((target) => ({
|
||||
entryId: target.messageId,
|
||||
message: {
|
||||
...(target.message as unknown as AgentMessage),
|
||||
idempotencyKey: target.request.idempotencyKey,
|
||||
content: target.request.state.persistedContent,
|
||||
} as unknown as AgentMessage,
|
||||
})),
|
||||
},
|
||||
});
|
||||
if (result.changed) {
|
||||
for (const target of rewriteTargets) {
|
||||
const rewritten =
|
||||
await findSourceReplyTranscriptMirrorByIdempotencyKey(
|
||||
resolvedTranscriptPath,
|
||||
target.request.idempotencyKey,
|
||||
);
|
||||
await attachSourceReplyManagedImages({
|
||||
messageId: rewritten?.messageId,
|
||||
request: target.request,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const sourceReplyContent = sourceReplyContentStates
|
||||
.flatMap((state) => {
|
||||
if (
|
||||
state.hasManagedOutgoingContent &&
|
||||
!state.backedManagedOutgoingContent
|
||||
) {
|
||||
const stripped = stripManagedOutgoingAssistantContentBlocks(
|
||||
state.broadcastContent,
|
||||
);
|
||||
return stripped?.length
|
||||
? stripped
|
||||
: [{ type: "text", text: "Media reply could not be displayed." }];
|
||||
}
|
||||
return state.broadcastContent;
|
||||
})
|
||||
.filter((block): block is AssistantDisplayContentBlock => Boolean(block));
|
||||
const sourceReplyTextFromContent =
|
||||
extractAssistantDisplayTextFromContent(sourceReplyContent);
|
||||
const sourceReplyText =
|
||||
sourceReplyTextFromContent ??
|
||||
(sourceReplyContent.length === 0 ? displayReply : undefined);
|
||||
const now = Date.now();
|
||||
const message = {
|
||||
role: "assistant",
|
||||
...(broadcastAssistantContent?.length
|
||||
? { content: broadcastAssistantContent }
|
||||
: displayReply
|
||||
? { content: [{ type: "text", text: displayReply }] }
|
||||
...(sourceReplyContent?.length
|
||||
? { content: sourceReplyContent }
|
||||
: sourceReplyText
|
||||
? { content: [{ type: "text", text: sourceReplyText }] }
|
||||
: {}),
|
||||
...(displayReply ? { text: displayReply } : {}),
|
||||
...(sourceReplyText ? { text: sourceReplyText } : {}),
|
||||
timestamp: now,
|
||||
stopReason: "stop",
|
||||
usage: { input: 0, output: 0, totalTokens: 0 },
|
||||
|
||||
Reference in New Issue
Block a user