diff --git a/src/agents/agent-command.live-model-switch.test.ts b/src/agents/agent-command.live-model-switch.test.ts index 0e11858eab7e..169d7c7107d2 100644 --- a/src/agents/agent-command.live-model-switch.test.ts +++ b/src/agents/agent-command.live-model-switch.test.ts @@ -1401,6 +1401,34 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { expect(state.assertLifecycleCurrentMock).toHaveBeenLastCalledWith("test-generation"); }); + it("persists structured transcript media for ACP turns", async () => { + state.acpResolveSessionMock.mockReturnValue({ + kind: "ready", + meta: { + agent: "claude", + cwd: "/tmp/workspace", + }, + }); + + await agentCommand({ + message: "[media attached: media://inbound/image-1]", + transcriptMessage: "", + transcriptMedia: [{ path: "/media/inbound/image-1.png", contentType: "image/png" }], + sessionKey: "agent:main:main", + }); + + expect(state.persistAcpTurnTranscriptMock).toHaveBeenCalledWith( + expect.objectContaining({ + transcriptBody: "", + userInput: { + text: "", + media: [{ path: "/media/inbound/image-1.png", contentType: "image/png" }], + mediaOnlyText: "[User sent media without caption]", + }, + }), + ); + }); + it("keeps the initial session touch for local runs", async () => { setupSingleAttemptFallback(); state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4")); @@ -3037,6 +3065,33 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { }); }); + it("persists structured transcript media without a caption", async () => { + setupSingleAttemptFallback(); + state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4")); + + await agentCommand({ + message: "[media attached: media://inbound/image-1]", + transcriptMessage: "", + transcriptMedia: [{ path: "/media/inbound/image-1.png", contentType: "image/png" }], + images: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }], + to: "+1234567890", + }); + + const attempt = mockCallArg(state.runAgentAttemptMock) as { + suppressPromptPersistenceOnRetry?: boolean; + userTurnTranscriptRecorder?: { message?: unknown }; + }; + expect(attempt.suppressPromptPersistenceOnRetry).toBe(false); + expect(attempt.userTurnTranscriptRecorder?.message).toMatchObject({ + role: "user", + content: "[User sent media without caption]", + MediaPath: "/media/inbound/image-1.png", + MediaPaths: ["/media/inbound/image-1.png"], + MediaType: "image/png", + MediaTypes: ["image/png"], + }); + }); + it("propagates non-switch errors without retrying and emits lifecycle error", async () => { state.runWithModelFallbackMock.mockRejectedValueOnce(new Error("provider down")); diff --git a/src/agents/agent-command.ts b/src/agents/agent-command.ts index 995095d1792b..db73916c8601 100644 --- a/src/agents/agent-command.ts +++ b/src/agents/agent-command.ts @@ -1155,6 +1155,15 @@ async function agentCommandInternal( const transcriptResult = await attemptExecutionRuntime.persistAcpTurnTranscript({ body, transcriptBody, + ...(opts.suppressPromptPersistence !== true && opts.transcriptMedia?.length + ? { + userInput: { + text: transcriptBody, + media: opts.transcriptMedia, + mediaOnlyText: "[User sent media without caption]", + }, + } + : {}), finalText: finalTextRaw, sessionId, sessionKey, @@ -1762,12 +1771,25 @@ async function agentCommandInternal( lifecycleEnded: false, }; const attemptLifecycleCallbacks = createAgentAttemptLifecycleCallbacks(attemptLifecycleState); + const transcriptMedia = opts.transcriptMedia ?? []; + const hasTranscriptMedia = transcriptMedia.length > 0; const suppressUserTurnPersistence = - opts.suppressPromptPersistence === true || opts.transcriptMessage === ""; + opts.suppressPromptPersistence === true || + (opts.transcriptMessage === "" && !hasTranscriptMedia); const recorderTranscriptText = transcriptBody || undefined; const userTurnTranscriptRecorder = createUserTurnTranscriptRecorder({ - ...(!suppressUserTurnPersistence && recorderTranscriptText - ? { input: { text: recorderTranscriptText } } + ...(!suppressUserTurnPersistence && (recorderTranscriptText || hasTranscriptMedia) + ? { + input: { + text: recorderTranscriptText, + ...(hasTranscriptMedia + ? { + media: transcriptMedia, + mediaOnlyText: "[User sent media without caption]", + } + : {}), + }, + } : {}), target: { transcriptPath: attemptSessionFile, diff --git a/src/agents/command/attempt-execution.cli.test.ts b/src/agents/command/attempt-execution.cli.test.ts index ea1ccb51e4fc..6c7b60767255 100644 --- a/src/agents/command/attempt-execution.cli.test.ts +++ b/src/agents/command/attempt-execution.cli.test.ts @@ -13,6 +13,7 @@ import { saveAuthProfileStore } from "../auth-profiles/store.js"; import type { EmbeddedAgentRunResult } from "../embedded-agent.js"; import { FailoverError } from "../failover-error.js"; import { + persistAcpTurnTranscript, persistCliTurnTranscript, runAgentAttempt as runAgentAttemptImpl, } from "./attempt-execution.js"; @@ -1374,6 +1375,46 @@ describe("CLI attempt execution", () => { ); }); + it("persists a media-only ACP user turn when the reply is empty", async () => { + const sessionKey = "agent:main:direct:acp-media-only"; + const sessionFile = path.join(tmpDir, "session-acp-media-only.jsonl"); + const sessionEntry: SessionEntry = { + sessionId: "session-acp-media-only", + sessionFile, + updatedAt: Date.now(), + }; + const sessionStore: Record = { [sessionKey]: sessionEntry }; + await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8"); + + await persistAcpTurnTranscript({ + body: "[media attached: media://inbound/image-1]", + transcriptBody: "", + userInput: { + text: "", + media: [{ path: "/media/inbound/image-1.png", contentType: "image/png" }], + mediaOnlyText: "[User sent media without caption]", + }, + finalText: "", + sessionId: sessionEntry.sessionId, + sessionKey, + sessionEntry, + sessionStore, + storePath, + sessionAgentId: "main", + sessionCwd: tmpDir, + config: {}, + }); + + expect(await readSessionMessages(sessionFile)).toContainEqual( + expect.objectContaining({ + role: "user", + content: "[User sent media without caption]", + MediaPath: "/media/inbound/image-1.png", + MediaType: "image/png", + }), + ); + }); + it("does not append a CLI transcript after the session is deleted", async () => { const sessionKey = "agent:main:subagent:cli-transcript-deleted"; const staleSessionFile = path.join(tmpDir, "session-cli-stale.jsonl"); diff --git a/src/agents/command/attempt-execution.ts b/src/agents/command/attempt-execution.ts index 704d22924a23..53d17fadd910 100644 --- a/src/agents/command/attempt-execution.ts +++ b/src/agents/command/attempt-execution.ts @@ -29,8 +29,10 @@ import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snaps import { isSubagentSessionKey } from "../../routing/session-key.js"; import { annotateInterSessionPromptText } from "../../sessions/input-provenance.js"; import { + buildPersistedUserTurnMessage, preparePersistedUserTurnMessageForTranscriptWrite, type PersistedUserTurnMessage, + type UserTurnInput, type UserTurnTranscriptRecorder, } from "../../sessions/user-turn-transcript.js"; import { buildWorkspaceSkillSnapshot } from "../../skills/loading/workspace.js"; @@ -308,11 +310,6 @@ async function persistTextTurnTranscript( ): Promise { const promptText = params.transcriptBody ?? params.body; const replyText = params.finalText; - if (!promptText && !replyText) { - return { kind: "persisted", sessionEntry: params.sessionEntry }; - } - - const messages = []; const userMessage = params.userMessage ?? (promptText @@ -322,6 +319,11 @@ async function persistTextTurnTranscript( timestamp: Date.now(), } as PersistedUserTurnMessage) : undefined); + if (!userMessage && !replyText) { + return { kind: "persisted", sessionEntry: params.sessionEntry }; + } + + const messages = []; if (userMessage) { messages.push({ message: userMessage, @@ -405,6 +407,7 @@ function isClaudeCliProvider(provider: string): boolean { export async function persistAcpTurnTranscript(params: { body: string; transcriptBody?: string; + userInput?: UserTurnInput; finalText: string; sessionId: string; sessionKey: string; @@ -418,6 +421,7 @@ export async function persistAcpTurnTranscript(params: { }): Promise { return await persistTextTurnTranscript({ ...params, + ...(params.userInput ? { userMessage: buildPersistedUserTurnMessage(params.userInput) } : {}), assistant: { api: "openai-responses", provider: "openclaw", diff --git a/src/agents/command/types.ts b/src/agents/command/types.ts index cbe184c39f01..a80de7feafba 100644 --- a/src/agents/command/types.ts +++ b/src/agents/command/types.ts @@ -10,6 +10,7 @@ import type { ChannelOutboundTargetMode } from "../../channels/plugins/types.pub import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; import type { PluginHookChannelContext } from "../../plugins/hook-types.js"; import type { InputProvenance } from "../../sessions/input-provenance.js"; +import type { UserTurnInput } from "../../sessions/user-turn-transcript.js"; import type { ExecElevatedDefaults } from "../bash-tools.exec-types.js"; import type { BootstrapContextRunKind } from "../bootstrap-mode.js"; import type { AgentStreamParams, ClientToolDefinition } from "./shared-types.js"; @@ -58,6 +59,8 @@ export type AgentCommandOpts = { message: string; /** User-visible transcript body; defaults to message and excludes runtime-only context. */ transcriptMessage?: string; + /** Durable media metadata for the user-visible transcript turn. */ + transcriptMedia?: UserTurnInput["media"]; /** Optional image attachments for multimodal messages. */ images?: ImageContent[]; /** Original inline/offloaded attachment order for inbound images. */ diff --git a/src/gateway/chat-attachments.test.ts b/src/gateway/chat-attachments.test.ts index 336b01dbcd98..e8d3960cc403 100644 --- a/src/gateway/chat-attachments.test.ts +++ b/src/gateway/chat-attachments.test.ts @@ -29,6 +29,7 @@ import { type ChatAttachment, DEFAULT_CHAT_ATTACHMENT_MAX_MB, parseMessageWithAttachments, + persistInboundImagesForTranscript, resolveChatAttachmentMaxBytes, UnsupportedAttachmentError, } from "./chat-attachments.js"; @@ -134,6 +135,48 @@ afterEach(() => { vi.restoreAllMocks(); }); +describe("persistInboundImagesForTranscript", () => { + it("preserves mixed image order and appends non-image offloads", async () => { + saveMediaBufferMock.mockResolvedValueOnce({ + id: "inline", + path: "/media/inbound/inline.jpg", + size: 5, + contentType: "image/jpeg", + }); + + const saved = await persistInboundImagesForTranscript({ + images: [{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }], + imageOrder: ["offloaded", "inline"], + offloadedRefs: [ + { + mediaRef: "media://inbound/offloaded", + id: "offloaded", + path: "/media/inbound/offloaded.png", + mimeType: "image/png", + label: "offloaded.png", + sizeBytes: 2_100_000, + }, + { + mediaRef: "media://inbound/report", + id: "report", + path: "/media/inbound/report.pdf", + mimeType: "application/pdf", + label: "report.pdf", + sizeBytes: 100, + }, + ], + log: { warn: vi.fn() }, + logContext: "test", + }); + + expect(saved.map((entry) => entry.path)).toEqual([ + "/media/inbound/offloaded.png", + "/media/inbound/inline.jpg", + "/media/inbound/report.pdf", + ]); + }); +}); + describe("parseMessageWithAttachments", () => { it("strips data URL prefix", async () => { const parsed = await parseMessageWithAttachments( diff --git a/src/gateway/chat-attachments.ts b/src/gateway/chat-attachments.ts index 5a5f9460c1ae..45c6d8fd87e6 100644 --- a/src/gateway/chat-attachments.ts +++ b/src/gateway/chat-attachments.ts @@ -8,7 +8,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; import type { PromptImageOrderEntry } from "../media/prompt-image-order.js"; import { sniffMimeFromBase64 } from "../media/sniff-mime-from-base64.js"; -import { deleteMediaBuffer, saveMediaBuffer } from "../media/store.js"; +import { deleteMediaBuffer, saveMediaBuffer, type SavedMedia } from "../media/store.js"; export type ChatAttachment = { type?: string; @@ -50,7 +50,7 @@ type NormalizedAttachment = { base64: string; }; -type SavedMedia = { +type SavedMediaRef = { id: string; path: string; }; @@ -60,6 +60,58 @@ const TEXT_ONLY_OFFLOAD_LIMIT = 10; export const DEFAULT_CHAT_ATTACHMENT_MAX_MB = 20; +export async function persistInboundImagesForTranscript(params: { + images: ChatImageContent[]; + imageOrder: PromptImageOrderEntry[]; + offloadedRefs: OffloadedRef[]; + log: Pick; + logContext: string; +}): Promise { + const inline: SavedMedia[] = []; + for (const image of params.images) { + try { + inline.push( + await saveMediaBuffer(Buffer.from(image.data, "base64"), image.mimeType, "inbound"), + ); + } catch (err) { + params.log.warn( + `${params.logContext}: failed to persist inbound image (${image.mimeType}): ${formatErrorMessage(err)}`, + ); + } + } + + const imageOffloaded: SavedMedia[] = []; + const nonImageOffloaded: SavedMedia[] = []; + for (const ref of params.offloadedRefs) { + const saved = { + id: ref.id, + path: ref.path, + size: ref.sizeBytes, + contentType: ref.mimeType, + }; + (ref.mimeType.startsWith("image/") ? imageOffloaded : nonImageOffloaded).push(saved); + } + if (params.imageOrder.length === 0) { + return [...inline, ...imageOffloaded, ...nonImageOffloaded]; + } + + const ordered: SavedMedia[] = []; + let inlineIndex = 0; + let offloadedIndex = 0; + for (const entry of params.imageOrder) { + const media = entry === "inline" ? inline[inlineIndex++] : imageOffloaded[offloadedIndex++]; + if (media) { + ordered.push(media); + } + } + ordered.push( + ...inline.slice(inlineIndex), + ...imageOffloaded.slice(offloadedIndex), + ...nonImageOffloaded, + ); + return ordered; +} + /** Resolve the maximum decoded attachment size accepted for chat image inputs. */ export function resolveChatAttachmentMaxBytes(cfg: OpenClawConfig): number { const configured = cfg.agents?.defaults?.mediaMaxMb; @@ -197,7 +249,7 @@ function ensureExtension(label: string, mime: string): string { return ext ? `${label}${ext}` : label; } -function assertSavedMedia(value: unknown, label: string): SavedMedia { +function assertSavedMedia(value: unknown, label: string): SavedMediaRef { if ( value === null || typeof value !== "object" || @@ -382,7 +434,7 @@ export async function parseMessageWithAttachments( const buffer = Buffer.from(b64, "base64"); verifyDecodedSize(buffer, sizeBytes, label); - let savedMedia: SavedMedia; + let savedMedia: SavedMediaRef; try { const labelWithExt = ensureExtension(label, finalMime); const rawResult = await saveMediaBuffer( diff --git a/src/gateway/server-methods/chat.ts b/src/gateway/server-methods/chat.ts index 1d8e01518d23..af7c23cac6d5 100644 --- a/src/gateway/server-methods/chat.ts +++ b/src/gateway/server-methods/chat.ts @@ -99,12 +99,7 @@ import { parseInboundMediaUri } from "../../media/media-reference.js"; import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; import { renderQrPngDataUrl } from "../../media/qr-image.js"; import { renderQrTerminal } from "../../media/qr-terminal.js"; -import { - deleteMediaBuffer, - MEDIA_MAX_BYTES, - type SavedMedia, - saveMediaBuffer, -} from "../../media/store.js"; +import { deleteMediaBuffer, MEDIA_MAX_BYTES, type SavedMedia } from "../../media/store.js"; import { createChannelMessageReplyPipeline } from "../../plugin-sdk/channel-outbound.js"; import type { ChannelRouteRef } from "../../plugin-sdk/channel-route.js"; import { isPluginOwnedSessionBindingRecord } from "../../plugins/conversation-binding.js"; @@ -148,6 +143,7 @@ import { MediaOffloadError, type OffloadedRef, parseMessageWithAttachments, + persistInboundImagesForTranscript, resolveChatAttachmentMaxBytes, UnsupportedAttachmentError, } from "../chat-attachments.js"; @@ -1333,73 +1329,13 @@ async function persistChatSendImages(params: { ) { return []; } - const inlineSaved: SavedMedia[] = []; - for (const img of params.images) { - try { - inlineSaved.push( - await saveMediaBuffer(Buffer.from(img.data, "base64"), img.mimeType, "inbound"), - ); - } catch (err) { - params.logGateway.warn( - `chat.send: failed to persist inbound image (${img.mimeType}): ${formatForLog(err)}`, - ); - } - } - // imageOrder now only tracks image slots (see chat-attachments.ts), so split - // offloaded refs by mime: image offloads interleave with inline images via - // imageOrder, and non-image offloads append to the transcript tail. Without - // this split a non-image file would consume the next image slot whenever - // both kinds appear in the same request. - const imageOffloadedSaved: SavedMedia[] = []; - const nonImageOffloadedSaved: SavedMedia[] = []; - for (const ref of params.offloadedRefs) { - const entry: SavedMedia = { - id: ref.id, - path: ref.path, - size: 0, - contentType: ref.mimeType, - }; - if (ref.mimeType.startsWith("image/")) { - imageOffloadedSaved.push(entry); - } else { - nonImageOffloadedSaved.push(entry); - } - } - if (params.imageOrder.length === 0) { - return [...inlineSaved, ...imageOffloadedSaved, ...nonImageOffloadedSaved]; - } - const saved: SavedMedia[] = []; - let inlineIndex = 0; - let offloadedIndex = 0; - for (const entry of params.imageOrder) { - if (entry === "inline") { - const inline = inlineSaved[inlineIndex++]; - if (inline) { - saved.push(inline); - } - continue; - } - const offloaded = imageOffloadedSaved[offloadedIndex++]; - if (offloaded) { - saved.push(offloaded); - } - } - for (; inlineIndex < inlineSaved.length; inlineIndex++) { - const inline = inlineSaved[inlineIndex]; - if (inline) { - saved.push(inline); - } - } - for (; offloadedIndex < imageOffloadedSaved.length; offloadedIndex++) { - const offloaded = imageOffloadedSaved[offloadedIndex]; - if (offloaded) { - saved.push(offloaded); - } - } - for (const offloaded of nonImageOffloadedSaved) { - saved.push(offloaded); - } - return saved; + return await persistInboundImagesForTranscript({ + images: params.images, + imageOrder: params.imageOrder, + offloadedRefs: params.offloadedRefs, + log: params.logGateway, + logContext: "chat.send", + }); } function stripTrailingOffloadedMediaMarkers(message: string, refs: OffloadedRef[]): string { diff --git a/src/gateway/server-node-events.runtime.ts b/src/gateway/server-node-events.runtime.ts index 7cda19e39ad7..fde348a3ad55 100644 --- a/src/gateway/server-node-events.runtime.ts +++ b/src/gateway/server-node-events.runtime.ts @@ -18,7 +18,11 @@ export { enqueueSystemEvent } from "../infra/system-events.js"; export { deleteMediaBuffer } from "../media/store.js"; export { normalizeMainKey, scopedHeartbeatWakeOptions } from "../routing/session-key.js"; export { defaultRuntime } from "../runtime.js"; -export { parseMessageWithAttachments, resolveChatAttachmentMaxBytes } from "./chat-attachments.js"; +export { + parseMessageWithAttachments, + persistInboundImagesForTranscript, + resolveChatAttachmentMaxBytes, +} from "./chat-attachments.js"; export { normalizeRpcAttachmentsToChatAttachments } from "./server-methods/attachment-normalize.js"; export { loadSessionEntry, diff --git a/src/gateway/server-node-events.test.ts b/src/gateway/server-node-events.test.ts index dc1cf984925a..b6ed6e4252a6 100644 --- a/src/gateway/server-node-events.test.ts +++ b/src/gateway/server-node-events.test.ts @@ -54,6 +54,7 @@ const loadOrCreateProcessDeviceIdentityMock = vi.hoisted(() => })), ); const parseMessageWithAttachmentsMock = vi.hoisted(() => vi.fn()); +const persistInboundImagesForTranscriptMock = vi.hoisted(() => vi.fn()); const normalizeChannelIdMock = vi.hoisted(() => vi.fn((channel?: string | null) => channel ?? null), ); @@ -123,6 +124,7 @@ const runtimeMocks = vi.hoisted(() => ({ model: entry?.model ?? "default-model", }), ), + persistInboundImagesForTranscript: persistInboundImagesForTranscriptMock, sanitizeInboundSystemTags: sanitizeInboundSystemTagsMock, scopedHeartbeatWakeOptions: vi.fn((sessionKey?: string, opts?: { reason: string }) => { const wakeOptions = { reason: opts?.reason }; @@ -267,6 +269,8 @@ describe("node exec events", () => { registerApnsRegistrationVi.mockClear(); loadOrCreateProcessDeviceIdentityMock.mockClear(); normalizeChannelIdVi.mockClear(); + persistInboundImagesForTranscriptMock.mockReset(); + persistInboundImagesForTranscriptMock.mockResolvedValue([]); normalizeChannelIdVi.mockImplementation((channel?: string | null) => channel ?? null); sanitizeInboundSystemTagsMock.mockClear(); updatePairedDeviceMetadataMock.mockClear(); @@ -1273,6 +1277,60 @@ describe("agent request events", () => { expectFields(parseCall?.[2], { supportsInlineImages: false }); }); + it("passes ordered durable media metadata to the agent transcript recorder", async () => { + parseMessageWithAttachmentsMock.mockResolvedValueOnce({ + message: "describe\n[media attached: media://inbound/offloaded]", + images: [{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }], + imageOrder: ["offloaded", "inline"], + offloadedRefs: [ + { + mediaRef: "media://inbound/offloaded", + id: "offloaded", + path: "/media/inbound/offloaded.png", + mimeType: "image/png", + label: "offloaded.png", + sizeBytes: 2_100_000, + }, + ], + }); + persistInboundImagesForTranscriptMock.mockResolvedValueOnce([ + { + id: "offloaded", + path: "/media/inbound/offloaded.png", + size: 2_100_000, + contentType: "image/png", + }, + { + id: "saved-inline", + path: "/media/inbound/saved-inline.jpg", + size: 5, + contentType: "image/jpeg", + }, + ]); + + await handleNodeEvent(buildCtx(), "node-media", { + event: "agent.request", + payloadJSON: JSON.stringify({ + message: "describe", + sessionKey: "agent:main:main", + attachments: [{ type: "image", mimeType: "image/png", content: "AAAA" }], + }), + }); + + expect(persistInboundImagesForTranscriptMock).toHaveBeenCalledWith( + expect.objectContaining({ imageOrder: ["offloaded", "inline"] }), + ); + expect(agentCommandMock).toHaveBeenCalledTimes(1); + expectFields(mockCallArg(agentCommandMock), { + message: "describe\n[media attached: media://inbound/offloaded]", + transcriptMessage: "describe", + transcriptMedia: [ + { path: "/media/inbound/offloaded.png", contentType: "image/png" }, + { path: "/media/inbound/saved-inline.jpg", contentType: "image/jpeg" }, + ], + }); + }); + it("declines non-image attachments cleanly when parse throws UnsupportedAttachmentError", async () => { const warn = vi.fn(); const ctx = buildCtx(); diff --git a/src/gateway/server-node-events.ts b/src/gateway/server-node-events.ts index f8fac609d329..d8529358c31d 100644 --- a/src/gateway/server-node-events.ts +++ b/src/gateway/server-node-events.ts @@ -43,6 +43,7 @@ import { resolveOutboundTarget, resolveSessionAgentId, resolveSessionModelRef, + persistInboundImagesForTranscript, sanitizeInboundSystemTags, sendDurableMessageBatch, canonicalizeSessionEntryAliases, @@ -478,11 +479,14 @@ export const handleNodeEvent = async ( const { storePath, entry, canonicalKey, storeKeys } = loadSessionEntry(sessionKey); let message = (link?.message ?? "").trim(); + const transcriptMessage = message; const normalizedAttachments = normalizeRpcAttachmentsToChatAttachments( link?.attachments ?? undefined, ); let images: Array<{ type: "image"; data: string; mimeType: string }> = []; let imageOrder: PromptImageOrderEntry[] = []; + let offloadedRefs: Awaited>["offloadedRefs"] = + []; if (!message && normalizedAttachments.length === 0) { return undefined; } @@ -510,6 +514,7 @@ export const handleNodeEvent = async ( message = parsed.message.trim(); images = parsed.images; imageOrder = parsed.imageOrder; + offloadedRefs = parsed.offloadedRefs; if (message.length > 20_000) { ctx.logGateway.warn( `agent.request message exceeds limit after attachment parsing (length=${message.length})`, @@ -596,11 +601,22 @@ export const handleNodeEvent = async ( ); } + const transcriptMedia = ( + await persistInboundImagesForTranscript({ + images, + imageOrder, + offloadedRefs, + log: ctx.logGateway, + logContext: "agent.request", + }) + ).map((media) => ({ path: media.path, contentType: media.contentType })); + dispatchNodeAgentCommand(ctx, nodeId, { runId: sessionId, message, images, imageOrder, + ...(transcriptMedia.length > 0 ? { transcriptMessage, transcriptMedia } : {}), sessionId, sessionKey: canonicalKey, thinking: link?.thinking ?? undefined, diff --git a/src/sessions/user-turn-transcript.ts b/src/sessions/user-turn-transcript.ts index 15f8d8ee9dd0..9d156a75cc24 100644 --- a/src/sessions/user-turn-transcript.ts +++ b/src/sessions/user-turn-transcript.ts @@ -258,7 +258,7 @@ function readOpenClawMessageMeta(message: AgentMessage): Record : undefined; } -function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedUserTurnMessage { +export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedUserTurnMessage { const mediaFields = buildPersistedUserTurnMediaFields(params.media); const hasMedia = Boolean(mediaFields.MediaPath); const text = normalizeTranscriptText(params.text);