diff --git a/src/gateway/chat-attachments.test.ts b/src/gateway/chat-attachments.test.ts index d84eb4e63d29..ac5d364a4a56 100644 --- a/src/gateway/chat-attachments.test.ts +++ b/src/gateway/chat-attachments.test.ts @@ -32,6 +32,7 @@ import { parseMessageWithAttachments, persistInboundImagesForTranscript, resolveChatAttachmentMaxBytes, + stripImageMediaMarkers, UnsupportedAttachmentError, } from "./chat-attachments.js"; @@ -233,7 +234,7 @@ describe("parseMessageWithAttachments", () => { expect(ref.label).toBe("report.pdf"); expect(ref.mediaRef).toMatch(/^media:\/\/inbound\//); expect(parsed.message).toBe(`read this\n[media attached: ${ref.mediaRef}]`); - expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message); + expect(stripImageMediaMarkers(parsed.message, parsed.offloadedRefs)).toBe(parsed.message); expect(saveMediaBufferMock).toHaveBeenCalledOnce(); expect(savedMime()).toBe("application/pdf"); expect(logs).toHaveLength(0); @@ -252,7 +253,7 @@ describe("parseMessageWithAttachments", () => { expect(parsed.message).toBe( `take a look\n[media attached: ${parsed.offloadedRefs[0]?.mediaRef}]`, ); - expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message); + expect(stripImageMediaMarkers(parsed.message, parsed.offloadedRefs)).toBe(parsed.message); expect(logs).toHaveLength(0); }); @@ -298,7 +299,9 @@ describe("parseMessageWithAttachments", () => { expect(parsed.message).toBe( `x\n[media attached: ${pdfRef.mediaRef}]\n[media attached: ${imageRef.mediaRef}]`, ); - expect(parsed.messageWithoutOffloadedImageRefs).toBe(`x\n[media attached: ${pdfRef.mediaRef}]`); + expect(stripImageMediaMarkers(parsed.message, parsed.offloadedRefs)).toBe( + `x\n[media attached: ${pdfRef.mediaRef}]`, + ); const trailingMediaLines = parsed.message .split("\n") .filter((line) => line.trim().startsWith("[media attached: media://inbound/")); @@ -445,7 +448,7 @@ describe("parseMessageWithAttachments validation errors", () => { it("passes through unchanged on text-only session with no attachments", async () => { const { parsed } = await parseWithWarnings("hello", [], { supportsInlineImages: false }); expect(parsed.message).toBe("hello"); - expect(parsed.messageWithoutOffloadedImageRefs).toBe("hello"); + expect(stripImageMediaMarkers(parsed.message, parsed.offloadedRefs)).toBe("hello"); expect(parsed.images).toHaveLength(0); expect(parsed.offloadedRefs).toHaveLength(0); expect(saveMediaBufferMock).not.toHaveBeenCalled(); @@ -474,7 +477,7 @@ describe("parseMessageWithAttachments validation errors", () => { expect(parsed.message).toBe( `read this\n[media attached: ${parsed.offloadedRefs[0]?.mediaRef}]`, ); - expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message); + expect(stripImageMediaMarkers(parsed.message, parsed.offloadedRefs)).toBe(parsed.message); } finally { await cleanupOffloadedRefs(parsed.offloadedRefs); } @@ -501,7 +504,7 @@ describe("parseMessageWithAttachments validation errors", () => { const offloaded = expectDefined(parsed.offloadedRefs[0], "offloaded image ref"); expect(offloaded.mimeType).toBe("image/png"); expect(parsed.message).toBe(`see this\n[media attached: ${offloaded.mediaRef}]`); - expect(parsed.messageWithoutOffloadedImageRefs).toBe("see this"); + expect(stripImageMediaMarkers(parsed.message, parsed.offloadedRefs)).toBe("see this"); expect(parsed.media).toEqual([ { path: offloaded.path, @@ -537,7 +540,7 @@ describe("parseMessageWithAttachments validation errors", () => { expect(parsed.message).toContain( "[image attachment omitted: text-only attachment limit reached]", ); - expect(parsed.messageWithoutOffloadedImageRefs).toBe( + expect(stripImageMediaMarkers(parsed.message, parsed.offloadedRefs)).toBe( "see these\n[image attachment omitted: text-only attachment limit reached]", ); expect(logs).toEqual([ diff --git a/src/gateway/chat-attachments.ts b/src/gateway/chat-attachments.ts index f48ba05860b6..fc9955aff130 100644 --- a/src/gateway/chat-attachments.ts +++ b/src/gateway/chat-attachments.ts @@ -36,7 +36,6 @@ export type OffloadedRef = { type ParsedMessageWithImages = { message: string; - messageWithoutOffloadedImageRefs: string; images: ChatImageContent[]; imageOrder: PromptImageOrderEntry[]; media: MediaFact[]; @@ -64,6 +63,16 @@ const TEXT_ONLY_OFFLOAD_LIMIT = 10; const DEFAULT_CHAT_ATTACHMENT_MAX_MB = 20; +export function stripImageMediaMarkers(message: string, refs: readonly OffloadedRef[]): string { + return refs.reduce((projected, ref) => { + const marker = ref.mimeType.startsWith("image/") ? `\n[media attached: ${ref.mediaRef}]` : ""; + const index = marker ? projected.lastIndexOf(marker) : -1; + return index < 0 + ? projected + : projected.slice(0, index) + projected.slice(index + marker.length); + }, message); +} + export async function persistInboundImagesForTranscript(params: { images: ChatImageContent[]; imageOrder: PromptImageOrderEntry[]; @@ -325,7 +334,6 @@ export async function parseMessageWithAttachments( if (!attachments || attachments.length === 0) { return { message, - messageWithoutOffloadedImageRefs: message, images: [], imageOrder: [], media: [], @@ -337,7 +345,6 @@ export async function parseMessageWithAttachments( const imageOrder: PromptImageOrderEntry[] = []; const offloadedRefs: OffloadedRef[] = []; let updatedMessage = message; - let messageWithoutOffloadedImageRefs = message; let textOnlyImageOffloadCount = 0; const savedMediaIds: string[] = []; @@ -431,8 +438,6 @@ export async function parseMessageWithAttachments( `${TEXT_ONLY_OFFLOAD_LIMIT} was reached`, ); updatedMessage += "\n[image attachment omitted: text-only attachment limit reached]"; - messageWithoutOffloadedImageRefs += - "\n[image attachment omitted: text-only attachment limit reached]"; continue; } @@ -469,11 +474,7 @@ export async function parseMessageWithAttachments( savedMediaIds.push(savedMedia.id); const mediaRef = `media://inbound/${savedMedia.id}`; - const mediaLine = `\n[media attached: ${mediaRef}]`; - updatedMessage += mediaLine; - if (!isImage) { - messageWithoutOffloadedImageRefs += mediaLine; - } + updatedMessage += `\n[media attached: ${mediaRef}]`; log?.info?.( shouldForceImageOffload && isImage ? `[Gateway] Offloaded image for text-only model. Saved: ${mediaRef}` @@ -504,10 +505,6 @@ export async function parseMessageWithAttachments( return { message: updatedMessage !== message ? updatedMessage.trimEnd() : message, - messageWithoutOffloadedImageRefs: - messageWithoutOffloadedImageRefs !== message - ? messageWithoutOffloadedImageRefs.trimEnd() - : message, images, imageOrder, media: offloadedRefs.map((ref) => ({ diff --git a/src/gateway/server-methods/chat-send-attachments.ts b/src/gateway/server-methods/chat-send-attachments.ts index fbf781feb925..e2101f1bea47 100644 --- a/src/gateway/server-methods/chat-send-attachments.ts +++ b/src/gateway/server-methods/chat-send-attachments.ts @@ -19,6 +19,7 @@ import { type OffloadedRef, parseMessageWithAttachments, resolveChatAttachmentMaxBytes, + stripImageMediaMarkers, UnsupportedAttachmentError, } from "../chat-attachments.js"; import { resolveGatewayModelSupportsImages } from "../session-utils.js"; @@ -239,16 +240,15 @@ export async function prepareChatSendAttachments(params: { supportsSessionModelImages || explicitOriginTargetsAcpSession(explicitOrigin) || explicitOriginTargetsPlugin; - const routeImageOffloadsAsMediaPaths = !supportsImages; const parsed = await parseMessageWithAttachments(inboundMessage, normalizedAttachments, { maxBytes: resolveChatAttachmentMaxBytes(cfg), log: context.logGateway, supportsImages, acceptNonImage: true, }); - parsedMessage = routeImageOffloadsAsMediaPaths - ? parsed.messageWithoutOffloadedImageRefs - : parsed.message; + parsedMessage = supportsImages + ? parsed.message + : stripImageMediaMarkers(parsed.message, parsed.offloadedRefs); parsedImages = parsed.images; imageOrder = parsed.imageOrder; offloadedRefs = parsed.offloadedRefs; @@ -258,7 +258,7 @@ export async function prepareChatSendAttachments(params: { workspaceDir: mediaPathOffloadWorkspaceDir, } = await prestageMediaPathOffloads({ offloadedRefs, - includeImageRefs: routeImageOffloadsAsMediaPaths, + includeImageRefs: !supportsImages, cfg, sessionKey, agentId,