From 91aee9cd51b0ec3c8d36325dae42fdea6f5f8943 Mon Sep 17 00:00:00 2001 From: Shakker Date: Mon, 25 May 2026 15:56:37 +0100 Subject: [PATCH] fix: keep media transcript text clean --- .../reply/get-reply-run.media-only.test.ts | 105 ++++++++++++++++++ src/auto-reply/reply/get-reply-run.ts | 18 ++- src/gateway/server-methods/chat.ts | 1 + src/sessions/user-turn-transcript.test.ts | 49 ++++++++ src/sessions/user-turn-transcript.ts | 53 +++++++++ 5 files changed, 225 insertions(+), 1 deletion(-) diff --git a/src/auto-reply/reply/get-reply-run.media-only.test.ts b/src/auto-reply/reply/get-reply-run.media-only.test.ts index 3a9db8926bfa..eaa8956ed638 100644 --- a/src/auto-reply/reply/get-reply-run.media-only.test.ts +++ b/src/auto-reply/reply/get-reply-run.media-only.test.ts @@ -971,6 +971,111 @@ describe("runPreparedReply media-only handling", () => { expect(call.followupRun.imageOrder).toEqual(["inline"]); }); + it("persists clean media captions instead of model-only media notes", async () => { + const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-")); + cleanupPaths.push(tmpDir); + const imagePath = path.join(tmpDir, "inbound.png"); + await writeFile( + imagePath, + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", + "base64", + ), + ); + + await runPreparedReply( + baseParams({ + ctx: { + Body: "What is in this image?", + RawBody: "What is in this image?", + CommandBody: "What is in this image?", + MediaPaths: [imagePath], + MediaTypes: ["image/png"], + MediaWorkspaceDir: tmpDir, + OriginatingChannel: "telegram", + OriginatingTo: "42", + ChatType: "direct", + }, + sessionCtx: { + Body: "[media attached: media://inbound/a.png (image/png)]\nTo send an image back, prefer the message tool (media/path/filePath).\nWhat is in this image?", + BodyStripped: + "[media attached: media://inbound/a.png (image/png)]\nTo send an image back, prefer the message tool (media/path/filePath).\nWhat is in this image?", + Provider: "telegram", + OriginatingChannel: "telegram", + OriginatingTo: "42", + ChatType: "direct", + MediaPaths: [imagePath], + MediaTypes: ["image/png"], + MediaWorkspaceDir: tmpDir, + }, + }), + ); + + const call = requireRunReplyAgentCall(); + expect(call.followupRun.userMessageForPersistence).toMatchObject({ + role: "user", + content: "What is in this image?", + MediaPath: imagePath, + MediaPaths: [imagePath], + MediaType: "image/png", + MediaTypes: ["image/png"], + }); + const persistedContent = call.followupRun.userMessageForPersistence?.content; + expect(persistedContent).toBe("What is in this image?"); + expect(persistedContent).not.toContain("media attached"); + expect(persistedContent).not.toContain("message tool"); + }); + + it("uses a media-only transcript label for exact media placeholders", async () => { + const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-")); + cleanupPaths.push(tmpDir); + const imagePath = path.join(tmpDir, "inbound.png"); + await writeFile( + imagePath, + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", + "base64", + ), + ); + + await runPreparedReply( + baseParams({ + ctx: { + Body: "", + RawBody: "", + CommandBody: "", + MediaPaths: [imagePath], + MediaTypes: ["image/png"], + MediaWorkspaceDir: tmpDir, + OriginatingChannel: "telegram", + OriginatingTo: "42", + ChatType: "direct", + }, + sessionCtx: { + Body: "", + BodyStripped: "", + Provider: "telegram", + OriginatingChannel: "telegram", + OriginatingTo: "42", + ChatType: "direct", + MediaPaths: [imagePath], + MediaTypes: ["image/png"], + MediaWorkspaceDir: tmpDir, + }, + }), + ); + + const call = requireRunReplyAgentCall(); + expect(call.followupRun.userMessageForPersistence).toMatchObject({ + role: "user", + content: "[User sent media without caption]", + MediaPath: imagePath, + MediaPaths: [imagePath], + MediaType: "image/png", + MediaTypes: ["image/png"], + }); + }); + it("does not rehydrate current MediaPaths after image understanding enriched the prompt", async () => { const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-")); cleanupPaths.push(tmpDir); diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index f27b1b1029c2..24b77e1af3cc 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -35,6 +35,7 @@ import { import { buildPersistedUserTurnMediaInputsFromFields, buildPersistedUserTurnMessage, + resolvePersistedUserTurnText, type UserTurnInput, } from "../../sessions/user-turn-transcript.js"; import { createLazyImportLoader } from "../../shared/lazy-promise.js"; @@ -1134,12 +1135,27 @@ export async function runPreparedReply( ? (internalOpts?.queuedFollowupAbortSignal ?? opts?.abortSignal) : undefined; const userTurnMediaForPersistence = buildPersistedUserTurnMediaInputsFromFields(ctx); + const userTurnTranscriptText = resolvePersistedUserTurnText( + { + Transcript: ctx.Transcript, + RawBody: ctx.RawBody, + CommandBody: ctx.CommandBody, + BodyForCommands: ctx.BodyForCommands, + Body: ctx.Body, + BodyStripped: sessionCtx.BodyStripped, + }, + { + hasMedia: userTurnMediaForPersistence.length > 0, + fallback: baseBodyTrimmedRaw, + }, + ); const userTurnInput = params.userTurnInput ?? (userTurnMediaForPersistence.length > 0 ? { - text: baseBodyTrimmedRaw, + text: userTurnTranscriptText, media: userTurnMediaForPersistence, + mediaOnlyText: "[User sent media without caption]", } : undefined); const userMessageForPersistence = userTurnInput diff --git a/src/gateway/server-methods/chat.ts b/src/gateway/server-methods/chat.ts index 39855fe3f858..0337a976a889 100644 --- a/src/gateway/server-methods/chat.ts +++ b/src/gateway/server-methods/chat.ts @@ -2578,6 +2578,7 @@ export const chatHandlers: GatewayRequestHandlers = { text: parsedMessage, media, timestamp: now, + mediaOnlyText: "[User sent media without caption]", } : undefined, ); diff --git a/src/sessions/user-turn-transcript.test.ts b/src/sessions/user-turn-transcript.test.ts index e9aa332632d0..a313ae824531 100644 --- a/src/sessions/user-turn-transcript.test.ts +++ b/src/sessions/user-turn-transcript.test.ts @@ -3,6 +3,7 @@ import { buildPersistedUserTurnMediaInputsFromFields, buildPersistedUserTurnMediaFields, buildPersistedUserTurnMessage, + resolvePersistedUserTurnText, } from "./user-turn-transcript.js"; describe("user turn transcript persistence", () => { @@ -180,4 +181,52 @@ describe("user turn transcript persistence", () => { }); }); }); + + describe("resolvePersistedUserTurnText", () => { + it("prefers clean inbound text over model prompt text", () => { + expect( + resolvePersistedUserTurnText( + { + RawBody: "What is in this image?", + BodyStripped: + "[media attached: media://inbound/a.png]\nTo send an image back, prefer the message tool.\nWhat is in this image?", + }, + { hasMedia: true }, + ), + ).toBe("What is in this image?"); + }); + + it("uses audio transcript before media placeholders", () => { + expect( + resolvePersistedUserTurnText( + { + Transcript: "please check this voice note", + RawBody: "", + CommandBody: "", + }, + { hasMedia: true }, + ), + ).toBe("please check this voice note"); + }); + + it("ignores exact generated media placeholders only when structured media is present", () => { + expect( + resolvePersistedUserTurnText( + { + RawBody: " (2 images)", + BodyStripped: " (2 images)", + }, + { hasMedia: true, fallback: "fallback" }, + ), + ).toBe("fallback"); + expect( + resolvePersistedUserTurnText( + { + RawBody: " (2 images)", + }, + { hasMedia: false }, + ), + ).toBe(" (2 images)"); + }); + }); }); diff --git a/src/sessions/user-turn-transcript.ts b/src/sessions/user-turn-transcript.ts index 12dfeb5db9d9..8aa898760e8b 100644 --- a/src/sessions/user-turn-transcript.ts +++ b/src/sessions/user-turn-transcript.ts @@ -26,6 +26,20 @@ export type UserTurnInput = { export type BuildPersistedUserTurnMessageParams = UserTurnInput; +export type PersistedUserTurnTextFieldSource = { + Transcript?: string | null; + RawBody?: string | null; + CommandBody?: string | null; + BodyForCommands?: string | null; + Body?: string | null; + BodyStripped?: string | null; +}; + +export type ResolvePersistedUserTurnTextOptions = { + hasMedia?: boolean; + fallback?: string | null; +}; + export type PersistedUserTurnMediaFieldSource = { MediaPath?: string | null; MediaPaths?: readonly (string | null | undefined)[] | null; @@ -44,6 +58,45 @@ function normalizeTranscriptText(value: string | null | undefined): string { return value ?? ""; } +const MEDIA_PLACEHOLDER_PATTERN = /^(?:\s+\([^)]*\))?$/i; + +function normalizePersistedUserTextCandidate( + value: string | null | undefined, + options: { hasMedia: boolean }, +): string | undefined { + const normalized = normalizeOptionalText(value); + if (!normalized) { + return undefined; + } + if (options.hasMedia && MEDIA_PLACEHOLDER_PATTERN.test(normalized)) { + return undefined; + } + return normalized; +} + +export function resolvePersistedUserTurnText( + fields: PersistedUserTurnTextFieldSource | null | undefined, + options: ResolvePersistedUserTurnTextOptions = {}, +): string | undefined { + const hasMedia = options.hasMedia === true; + const candidates = [ + fields?.Transcript, + fields?.RawBody, + fields?.CommandBody, + fields?.BodyForCommands, + fields?.Body, + fields?.BodyStripped, + options.fallback, + ]; + for (const candidate of candidates) { + const normalized = normalizePersistedUserTextCandidate(candidate, { hasMedia }); + if (normalized) { + return normalized; + } + } + return undefined; +} + function mediaTypeForTranscript(media: PersistedUserTurnMediaInput): string { return ( normalizeOptionalText(media.contentType) ??