From f6818ae5eabc3bf1d44b47ac46e7b57eb8d70f2c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 23 Jul 2026 03:11:16 -0400 Subject: [PATCH] refactor(media): hydrate and prune images from structured facts, delete marker-text parsing (#112913) * refactor(media): hydrate and prune images from structured facts, delete marker-text parsing Embedded, CLI, host-context replay, and plugin-harness image hydration now consume the persisted/runtime MediaFact carrier instead of parsing OpenClaw-authored [media attached:]/[Image: source:] marker text. The prompt-text attachment parser family (splitPromptAndAttachmentRefs, leading/ trailing marker extractors, ref-count consumption, text-based prune scrubbing) is deleted; explicit user-authored paths and file URLs still hydrate. - Described-image suppression is a declared MediaFact.hydrationSuppressed field carried across copy/reprojection boundaries; the runtime symbol plumbing is removed. - Persisted facts plus mediaImageLayout are authoritative at every hydration entry point; image blocks reconcile by structural fact index, never byte/MIME equality. - Offloaded attachments (image and non-image) persist their media://inbound claim-check alias so ownership-aware history pruning can scrub markers after persistence; factless pre-MediaFact rows keep narrow legacy redaction. - History pruning removes facts, layout, and block provenance structurally before replay. Prompt bytes are golden-equal throughout; goldens from the fact-carrier PR are untouched. 1,303 tests across 44 files; delegated check:changed green (run 29980534186, pinned to the declared merge-base). * fix(media): fail missing inline image slots * test(agents): carry offloaded CLI retry image as a structured fact The retry-image reliability test modeled an offloaded attachment with only its [media attached:] marker text; offloaded attachments are carried as structured media facts and marker text is presentation-only, never parsed for hydration. --- src/agents/cli-runner.helpers.test.ts | 141 +-- src/agents/cli-runner.reliability.test.ts | 3 + src/agents/cli-runner.spawn.test.ts | 41 + src/agents/cli-runner/execute.ts | 12 +- src/agents/cli-runner/helpers.ts | 80 +- .../cli-backend-dispatch.ts | 1 + .../run/attempt-context-guards.ts | 16 + .../run/attempt-prompt-dispatch.test.ts | 4 + .../attempt-prompt-execution-prepare.test.ts | 88 +- .../run/attempt-prompt-execution-prepare.ts | 28 +- .../run/attempt-session-runtime-prepare.ts | 3 + .../run/attempt-setup.ts | 94 +- .../run/attempt.queue-message.test.ts | 8 +- .../run/attempt.queue-message.ts | 9 +- .../attempt.spawn-workspace.test-support.ts | 2 + .../embedded-agent-runner/run/attempt.ts | 1 + .../run/history-image-prune.test.ts | 372 +++++++- .../run/history-image-prune.ts | 216 ++++- .../run/images.combinations.test.ts | 157 ++++ .../run/images.media-refs.test.ts | 293 +++++++ .../run/images.media-refs.ts | 160 ++++ .../run/images.replay.test.ts | 680 +++++++++++++++ .../embedded-agent-runner/run/images.test.ts | 583 ++++++++----- .../embedded-agent-runner/run/images.ts | 806 +++++++++--------- .../run/plugin-harness-prompt-images.ts | 113 +++ .../run/prompt-image-metadata.ts | 118 +++ .../run/run-attempt-dispatch.media.test.ts | 303 +++++++ .../run/run-attempt-dispatch.ts | 11 +- .../sessions/agent-session-prompting.ts | 34 +- src/agents/sessions/sdk.test.ts | 19 +- src/auto-reply/media-note.test.ts | 15 +- src/auto-reply/media-note.ts | 30 +- src/auto-reply/reply/agent-runner.ts | 1 + src/auto-reply/reply/current-turn-images.ts | 8 +- .../reply/get-reply-run.media-only.test.ts | 17 + src/auto-reply/reply/get-reply-run.ts | 81 +- src/auto-reply/reply/reply-run-registry.ts | 2 + src/gateway/chat-attachments.test.ts | 40 +- src/gateway/chat-attachments.ts | 23 +- .../server-methods/chat-send-attachments.ts | 28 +- .../chat-send-user-turn.test.ts | 176 +++- .../server-methods/chat-send-user-turn.ts | 44 +- .../chat.directive-tags.test.ts | 25 +- src/media/media-facts.ts | 66 +- src/media/runtime-prompt-image-provenance.ts | 45 + .../user-turn-transcript.media-normalize.ts | 97 +++ src/sessions/user-turn-transcript.ts | 59 +- src/sessions/user-turn-transcript.types.ts | 13 +- 48 files changed, 4225 insertions(+), 941 deletions(-) create mode 100644 src/agents/embedded-agent-runner/run/images.combinations.test.ts create mode 100644 src/agents/embedded-agent-runner/run/images.media-refs.test.ts create mode 100644 src/agents/embedded-agent-runner/run/images.media-refs.ts create mode 100644 src/agents/embedded-agent-runner/run/images.replay.test.ts create mode 100644 src/agents/embedded-agent-runner/run/plugin-harness-prompt-images.ts create mode 100644 src/agents/embedded-agent-runner/run/prompt-image-metadata.ts create mode 100644 src/agents/embedded-agent-runner/run/run-attempt-dispatch.media.test.ts create mode 100644 src/media/runtime-prompt-image-provenance.ts create mode 100644 src/sessions/user-turn-transcript.media-normalize.ts diff --git a/src/agents/cli-runner.helpers.test.ts b/src/agents/cli-runner.helpers.test.ts index a0a362f59cd9..47ee143d449b 100644 --- a/src/agents/cli-runner.helpers.test.ts +++ b/src/agents/cli-runner.helpers.test.ts @@ -3,11 +3,11 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "@openclaw/ai/internal/shared"; -import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants"; import { expectDefined } from "@openclaw/normalization-core"; import type { ImageContent } from "openclaw/plugin-sdk/llm"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { createSolidPngBuffer } from "../../test/helpers/image-fixtures.js"; +import { buildInboundMediaNoteProjection } from "../auto-reply/media-note.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; import { escapeRegExp } from "../shared/regexp.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; @@ -28,7 +28,7 @@ describe("prepareCliPromptImagePayload prompt references", () => { }); it("returns empty results when the prompt has no image refs", async () => { - const loadImageFromRefSpy = vi.spyOn(promptImageUtils, "loadImageFromRef"); + const detectAndLoadPromptImagesSpy = vi.spyOn(promptImageUtils, "detectAndLoadPromptImages"); const sanitizeImageBlocksSpy = vi.spyOn(toolImages, "sanitizeImageBlocks"); await expect( @@ -39,12 +39,12 @@ describe("prepareCliPromptImagePayload prompt references", () => { }), ).resolves.toStrictEqual({ prompt: "just text" }); - expect(loadImageFromRefSpy).not.toHaveBeenCalled(); + expect(detectAndLoadPromptImagesSpy).not.toHaveBeenCalled(); expect(sanitizeImageBlocksSpy).not.toHaveBeenCalled(); }); it("does not reload OpenClaw CLI image cache paths from prior prompt text", async () => { - const loadImageFromRefSpy = vi.spyOn(promptImageUtils, "loadImageFromRef"); + const detectAndLoadPromptImagesSpy = vi.spyOn(promptImageUtils, "detectAndLoadPromptImages"); const sanitizeImageBlocksSpy = vi.spyOn(toolImages, "sanitizeImageBlocks"); await expect( @@ -59,99 +59,115 @@ describe("prepareCliPromptImagePayload prompt references", () => { }); // Cached image paths are generated output, not fresh user references. - expect(loadImageFromRefSpy).not.toHaveBeenCalled(); + expect(detectAndLoadPromptImagesSpy).not.toHaveBeenCalled(); expect(sanitizeImageBlocksSpy).not.toHaveBeenCalled(); }); - it("passes the max-byte guardrail through load and sanitize", async () => { - const loadedImage: ImageContent = { - type: "image", - data: "c29tZS1pbWFnZQ==", - mimeType: "image/png", - }; - const sanitizedImage: ImageContent = { - type: "image", - data: "c2FuaXRpemVkLWltYWdl", - mimeType: "image/jpeg", - }; + it("hydrates explicit prompt refs through the shared image loader", async () => { const workspaceDir = await fs.mkdtemp( path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-ref-image-"), ); - - const loadImageFromRefSpy = vi - .spyOn(promptImageUtils, "loadImageFromRef") - .mockResolvedValueOnce(loadedImage); - const sanitizeImageBlocksSpy = vi - .spyOn(toolImages, "sanitizeImageBlocks") - .mockResolvedValueOnce({ images: [sanitizedImage], dropped: 0 }); + const imagePath = path.join(workspaceDir, "photo.png"); + const image = createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 }); + await fs.writeFile(imagePath, image); try { const result = await prepareCliPromptImagePayload({ backend: { command: "gemini", imagePathScope: "workspace" }, - prompt: "Look at /tmp/photo.png", + prompt: `Look at ${imagePath}`, workspaceDir, }); - const [ref, loadedWorkspaceDir, options] = loadImageFromRefSpy.mock.calls[0] ?? []; - expect(ref?.resolved).toBe("/tmp/photo.png"); - expect(ref?.type).toBe("path"); - expect(loadedWorkspaceDir).toBe(workspaceDir); - expect(options).toEqual({ - maxBytes: MAX_IMAGE_BYTES, - workspaceOnly: undefined, - sandbox: undefined, - }); - expect(sanitizeImageBlocksSpy).toHaveBeenCalledWith([loadedImage], "prompt:images", { - maxBytes: MAX_IMAGE_BYTES, - }); expect(result.imagePaths).toHaveLength(1); await expect( fs.readFile(expectDefined(result.imagePaths?.[0], "image path")), - ).resolves.toEqual(Buffer.from(sanitizedImage.data, "base64")); + ).resolves.toEqual(image); } finally { await fs.rm(workspaceDir, { recursive: true, force: true }); } }); it("dedupes repeated refs and skips failed loads before sanitizing", async () => { - const loadedImage: ImageContent = { - type: "image", - data: "b25lLWltYWdl", - mimeType: "image/png", - }; - - const loadImageFromRefSpy = vi - .spyOn(promptImageUtils, "loadImageFromRef") - .mockResolvedValueOnce(loadedImage) - .mockResolvedValueOnce(null); - const sanitizeImageBlocksSpy = vi - .spyOn(toolImages, "sanitizeImageBlocks") - .mockResolvedValueOnce({ images: [loadedImage], dropped: 0 }); - const workspaceDir = await fs.mkdtemp( path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-ref-dedupe-"), ); + const imagePath = path.join(workspaceDir, "a.png"); + await fs.writeFile(imagePath, createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 })); try { const result = await prepareCliPromptImagePayload({ backend: { command: "gemini", imagePathScope: "workspace" }, - prompt: "Compare /tmp/a.png with /tmp/a.png and /tmp/b.png", + prompt: `Compare ${imagePath} with ${imagePath} and ${path.join(workspaceDir, "missing.png")}`, workspaceDir, }); - expect(loadImageFromRefSpy).toHaveBeenCalledTimes(2); - expect( - loadImageFromRefSpy.mock.calls.map( - (call) => (call[0] as { resolved?: string } | undefined)?.resolved, - ), - ).toEqual(["/tmp/a.png", "/tmp/b.png"]); - expect(sanitizeImageBlocksSpy).toHaveBeenCalledWith([loadedImage], "prompt:images", { - maxBytes: MAX_IMAGE_BYTES, - }); expect(result.imagePaths).toHaveLength(1); } finally { await fs.rm(workspaceDir, { recursive: true, force: true }); } }); + + it("surfaces structured image hydration failures", async () => { + const workspaceDir = await fs.mkdtemp( + path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-structured-failure-"), + ); + try { + await expect( + prepareCliPromptImagePayload({ + backend: { command: "codex" }, + prompt: "describe the attachment", + workspaceDir, + media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }], + }), + ).rejects.toThrow("failed to hydrate 1 structured image attachment"); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("surfaces inline sanitization failure when a preceding image fact is suppressed", async () => { + await expect( + prepareCliPromptImagePayload({ + backend: { command: "codex" }, + prompt: "already described", + workspaceDir: "/tmp", + images: [{ type: "image", data: "%%%", mimeType: "image/png" }], + imageOrder: ["inline"], + media: [ + { + path: "/tmp/described-missing.png", + contentType: "image/png", + hydrationSuppressed: true, + }, + { path: "/tmp/inline.png", contentType: "image/png" }, + ], + }), + ).rejects.toThrow("failed to hydrate 1 structured image attachment"); + }); + + it("accepts an intentionally non-hydrating remote-only image fact", async () => { + const media = buildInboundMediaNoteProjection({ + MediaPaths: [""], + MediaUrls: ["https://example.com/described.png"], + MediaTypes: ["image/png"], + MediaUnderstanding: [ + { + kind: "image.description", + attachmentIndex: 0, + text: "already described", + provider: "test", + }, + ], + }).media; + + await expect( + prepareCliPromptImagePayload({ + backend: { command: "codex" }, + prompt: "already described", + workspaceDir: "/tmp", + media, + }), + ).resolves.toEqual({ prompt: "already described" }); + }); }); describe("buildCliArgs", () => { @@ -366,6 +382,7 @@ describe("writeCliImages", () => { input: "arg", }, prompt: `[media attached: ${sourceImage} (image/png)]\n\n`, + media: [{ path: sourceImage, contentType: "image/png" }], workspaceDir: tempDir, }); const argv = buildCliArgs({ @@ -411,6 +428,7 @@ describe("writeCliImages", () => { input: "stdin", }, prompt, + media: [{ path: sourceImage, contentType: "image/png" }], workspaceDir: tempDir, }); const promptWithImages = prepared.prompt; @@ -560,6 +578,7 @@ describe("writeCliImages", () => { }, ], imageOrder: ["offloaded", "inline"], + media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }], }); expect(prepared.imagePaths).toHaveLength(2); diff --git a/src/agents/cli-runner.reliability.test.ts b/src/agents/cli-runner.reliability.test.ts index 6158ec35c6dd..868e5e0088a8 100644 --- a/src/agents/cli-runner.reliability.test.ts +++ b/src/agents/cli-runner.reliability.test.ts @@ -689,6 +689,9 @@ describe("runCliAgent reliability", () => { }, ], imageOrder: ["offloaded", "inline"], + // Offloaded attachments are carried as structured facts; the trailing + // marker text is presentation only and is never parsed for hydration. + media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }], }; const result = await runPreparedCliAgent(context); diff --git a/src/agents/cli-runner.spawn.test.ts b/src/agents/cli-runner.spawn.test.ts index 8bed21e6d9c2..e697c3bdc45f 100644 --- a/src/agents/cli-runner.spawn.test.ts +++ b/src/agents/cli-runner.spawn.test.ts @@ -593,9 +593,50 @@ describe("runCliAgent spawn path", () => { await expect(executePreparedCliRun(context)).rejects.toThrow( "paired-node Claude CLI sessions do not support attachments or images", ); + context.params.imagePrompt = undefined; + context.params.media = [{ path: "/tmp/hydratable.png", kind: "image" }]; + await expect(executePreparedCliRun(context)).rejects.toThrow( + "paired-node Claude CLI sessions do not support attachments or images", + ); expect(invokeNode).not.toHaveBeenCalled(); }); + it("allows non-hydratable image facts on a text-only node turn", async () => { + const invokeNode = vi.fn(async (params: Parameters[0]) => { + params.onProgress( + [ + JSON.stringify({ type: "system", subtype: "init", session_id: "node-text-only" }), + JSON.stringify({ type: "result", session_id: "node-text-only", result: "ok" }), + "", + ].join("\n"), + ); + return { + ok: true, + payloadJSON: JSON.stringify({ exitCode: 0, stderrTail: "", truncated: false }), + }; + }); + setCliRunnerExecuteTestDeps({ invokeNodeClaudeCliRun: invokeNode }); + const context = buildPreparedCliRunContext({ + provider: "claude-cli", + model: "claude-opus-4-8", + runId: "run-node-text-only-media-facts", + prompt: "already described", + sessionEntry: { + sessionId: "openclaw-session", + updatedAt: 1, + execHost: "node", + execNode: "node-a", + }, + }); + context.params.media = [ + { kind: "image" }, + { kind: "image", url: "https://example.test/described.png" }, + ]; + + await expect(executePreparedCliRun(context)).resolves.toMatchObject({ text: "ok" }); + expect(invokeNode).toHaveBeenCalledOnce(); + }); + it("does not inject hardcoded 'Tools are disabled' text into CLI arguments", async () => { supervisorSpawnMock.mockResolvedValueOnce( createManagedRun({ diff --git a/src/agents/cli-runner/execute.ts b/src/agents/cli-runner/execute.ts index 9934abf2576e..6c468c5bcb04 100644 --- a/src/agents/cli-runner/execute.ts +++ b/src/agents/cli-runner/execute.ts @@ -68,6 +68,10 @@ import type { MessagingToolSend, MessagingToolSourceReplyPayload, } from "../embedded-agent-messaging.types.js"; +import { + detectImageReferences, + hasHydratableMediaImages, +} from "../embedded-agent-runner/run/images.js"; import { extractMessagingToolSendResult, extractMessagingToolSourceReplyPayload, @@ -458,7 +462,12 @@ export async function executePreparedCliRun( }), context.backendResolved.textTransforms?.input, ); - if (nodePlacement && ((params.images?.length ?? 0) > 0 || Boolean(params.imagePrompt?.trim()))) { + if ( + nodePlacement && + ((params.images?.length ?? 0) > 0 || + hasHydratableMediaImages(params.media) || + (params.imagePrompt ? detectImageReferences(params.imagePrompt).length > 0 : false)) + ) { throw new Error("paired-node Claude CLI sessions do not support attachments or images"); } const { @@ -474,6 +483,7 @@ export async function executePreparedCliRun( workspaceDir: context.workspaceDir, images: params.images, imageOrder: params.imageOrder, + media: params.media, }); prompt = promptWithImages; diff --git a/src/agents/cli-runner/helpers.ts b/src/agents/cli-runner/helpers.ts index 164b77e3f426..2f911836bfd9 100644 --- a/src/agents/cli-runner/helpers.ts +++ b/src/agents/cli-runner/helpers.ts @@ -23,6 +23,7 @@ import { privateFileStore } from "../../infra/private-file-store.js"; import { tempWorkspace } from "../../infra/private-temp-workspace.js"; import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js"; import type { ImageContent } from "../../llm/types.js"; +import type { MediaFact } from "../../media/media-facts.js"; import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; import { KeyedAsyncQueue } from "../../plugin-sdk/keyed-async-queue.js"; import type { CliBackendConfig } from "../../plugins/cli-backend.types.js"; @@ -32,16 +33,13 @@ import type { EmbeddedContextFile } from "../embedded-agent-helpers.js"; import { detectAndLoadPromptImages, detectImageReferences, - loadImageFromRef, } from "../embedded-agent-runner/run/images.js"; import { resolveDefaultModelForAgent } from "../model-selection.js"; import type { AgentTool } from "../runtime/index.js"; -import type { SandboxFsBridge } from "../sandbox/fs-bridge.js"; import { detectRuntimeShell } from "../shell-utils.js"; import { buildConfiguredAgentSystemPrompt } from "../system-prompt-config.js"; import { buildSystemPromptParams } from "../system-prompt-params.js"; import type { SilentReplyPromptMode } from "../system-prompt.types.js"; -import { sanitizeImageBlocks } from "../tool-images.js"; import { cliBackendLog } from "./log.js"; import { formatTomlConfigOverride } from "./toml-inline.js"; /** Re-export CLI reliability helpers used by older runner call sites. */ @@ -362,44 +360,6 @@ function appendImagePathsToPrompt(prompt: string, paths: string[], prefix = ""): return `${trimmed}${separator}${paths.map((entry) => `${prefix}${entry}`).join("\n")}`; } -/** Loads and sanitizes image references found in prompt text. */ -async function loadPromptRefImages(params: { - prompt: string; - workspaceDir: string; - maxBytes?: number; - workspaceOnly?: boolean; - sandbox?: { root: string; bridge: SandboxFsBridge }; -}): Promise { - const refs = detectImageReferences(params.prompt); - if (refs.length === 0) { - return []; - } - - const maxBytes = params.maxBytes ?? MAX_IMAGE_BYTES; - const seen = new Set(); - const images: ImageContent[] = []; - for (const ref of refs) { - const key = `${ref.type}:${ref.resolved}`; - if (seen.has(key)) { - continue; - } - seen.add(key); - const image = await loadImageFromRef(ref, params.workspaceDir, { - maxBytes, - workspaceOnly: params.workspaceOnly, - sandbox: params.sandbox, - }); - if (image) { - images.push(image); - } - } - - const { images: sanitizedImages } = await sanitizeImageBlocks(images, "prompt:images", { - maxBytes, - }); - return sanitizedImages; -} - /** Writes CLI image payloads to private paths and returns their file paths. */ async function writeCliImages(params: { backend: CliBackendConfig; @@ -459,27 +419,35 @@ export async function prepareCliPromptImagePayload(params: { workspaceDir: string; images?: ImageContent[]; imageOrder?: PromptImageOrderEntry[]; + media?: MediaFact[]; }): Promise<{ prompt: string; imagePaths?: string[]; cleanupImages?: () => Promise; }> { let prompt = params.prompt; - const resolvedImages = - params.imagePrompt !== undefined - ? ( - await detectAndLoadPromptImages({ - prompt: params.imagePrompt, - workspaceDir: params.workspaceDir, - model: { input: ["text", "image"] }, - existingImages: params.images, - imageOrder: params.imageOrder, - maxBytes: MAX_IMAGE_BYTES, - }) - ).images - : params.images && params.images.length > 0 - ? params.images - : await loadPromptRefImages({ prompt, workspaceDir: params.workspaceDir }); + const imagePrompt = params.imagePrompt ?? prompt; + const needsHydration = + params.imagePrompt !== undefined || + Boolean(params.media?.length) || + (!params.images?.length && detectImageReferences(imagePrompt).length > 0); + const imageResult = needsHydration + ? await detectAndLoadPromptImages({ + prompt: imagePrompt, + media: params.media, + workspaceDir: params.workspaceDir, + model: { input: ["text", "image"] }, + existingImages: params.images, + imageOrder: params.imageOrder, + maxBytes: MAX_IMAGE_BYTES, + }) + : undefined; + if (imageResult?.failedMediaCount) { + throw new Error( + `failed to hydrate ${imageResult.failedMediaCount} structured image attachment(s) for CLI input`, + ); + } + const resolvedImages = imageResult?.images ?? params.images ?? []; if (resolvedImages.length === 0) { return { prompt }; } diff --git a/src/agents/embedded-agent-runner/cli-backend-dispatch.ts b/src/agents/embedded-agent-runner/cli-backend-dispatch.ts index 58dda5862c15..1bc8a10c0672 100644 --- a/src/agents/embedded-agent-runner/cli-backend-dispatch.ts +++ b/src/agents/embedded-agent-runner/cli-backend-dispatch.ts @@ -193,6 +193,7 @@ async function runEmbeddedAgentViaCliBackend( agentDir: params.agentDir, config: params.config, prompt: params.prompt, + imagePrompt: params.prompt, media: params.media, provider: dispatch.provider, model: params.model, diff --git a/src/agents/embedded-agent-runner/run/attempt-context-guards.ts b/src/agents/embedded-agent-runner/run/attempt-context-guards.ts index 9d48730b38c8..663c36a9a560 100644 --- a/src/agents/embedded-agent-runner/run/attempt-context-guards.ts +++ b/src/agents/embedded-agent-runner/run/attempt-context-guards.ts @@ -1,9 +1,12 @@ /** Installs attempt-local context engine, tool-result, image, and frame guards. */ +import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants"; import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../../context-engine/host-compat.js"; import { buildContextEngineRuntimeSettings } from "../../../context-engine/runtime-settings.js"; import type { ContextEngine } from "../../../context-engine/types.js"; import { isHeartbeatLifecycleRunKind } from "../../bootstrap-mode.js"; import { DEFAULT_CONTEXT_TOKENS } from "../../defaults.js"; +import { resolveImageSanitizationLimits } from "../../image-sanitization.js"; +import type { SandboxContext } from "../../sandbox/types.js"; import type { guardSessionManager } from "../../session-tool-result-guard-wrapper.js"; import type { AgentSession } from "../../sessions/index.js"; import { invalidateComputerFrameIfMissing } from "../../tools/computer-tool.js"; @@ -29,6 +32,7 @@ export function installEmbeddedAttemptContextGuards(input: { attempt: EmbeddedRunAttemptParams; computerContextEpoch: { value: number }; effectiveCwd: string; + effectiveFsWorkspaceOnly: boolean; effectiveWorkspace: string; getPrePromptMessageCount: () => number; getPromptCache: () => EmbeddedRunAttemptResult["promptCache"]; @@ -39,6 +43,7 @@ export function installEmbeddedAttemptContextGuards(input: { sessionAgentId: string; sessionManager: ReturnType; settingsManager: AgentSession["settingsManager"]; + sandbox?: SandboxContext | null; }): { getAfterTurnCheckpoint: () => number | null; remove: () => void; @@ -152,6 +157,17 @@ export function installEmbeddedAttemptContextGuards(input: { const removeHistoryImagePruneContextTransform = installHistoryImagePruneContextTransform( activeSession.agent, + { + workspaceDir: input.effectiveWorkspace, + model: attempt.model, + maxBytes: MAX_IMAGE_BYTES, + maxDimensionPx: resolveImageSanitizationLimits(attempt.config).maxDimensionPx, + workspaceOnly: input.effectiveFsWorkspaceOnly, + sandbox: + input.sandbox?.enabled && input.sandbox.fsBridge + ? { root: input.sandbox.workspaceDir, bridge: input.sandbox.fsBridge } + : undefined, + }, ); const previousComputerFrameTransform = activeSession.agent.transformContext; activeSession.agent.transformContext = async (messages, signal) => { diff --git a/src/agents/embedded-agent-runner/run/attempt-prompt-dispatch.test.ts b/src/agents/embedded-agent-runner/run/attempt-prompt-dispatch.test.ts index c0f676307854..961cea056806 100644 --- a/src/agents/embedded-agent-runner/run/attempt-prompt-dispatch.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-prompt-dispatch.test.ts @@ -65,7 +65,9 @@ describe("dispatchEmbeddedAttemptPrompt", () => { vi.clearAllMocks(); hoisted.prepareEmbeddedAttemptPromptExecution.mockResolvedValue({ images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }], + imageFactIndexes: [null], detectedRefs: [], + failedMediaCount: 0, loadedCount: 1, skippedCount: 0, }); @@ -82,7 +84,9 @@ describe("dispatchEmbeddedAttemptPrompt", () => { order.push("images"); return { images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }], + imageFactIndexes: [null], detectedRefs: [], + failedMediaCount: 0, loadedCount: 1, skippedCount: 0, }; diff --git a/src/agents/embedded-agent-runner/run/attempt-prompt-execution-prepare.test.ts b/src/agents/embedded-agent-runner/run/attempt-prompt-execution-prepare.test.ts index f1494f91529a..49b181e5022e 100644 --- a/src/agents/embedded-agent-runner/run/attempt-prompt-execution-prepare.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-prompt-execution-prepare.test.ts @@ -12,7 +12,10 @@ const hoisted = vi.hoisted(() => ({ withSessionWriteLock: vi.fn(async (operation: () => unknown) => await operation()), })); -vi.mock("@openclaw/media-core/constants", () => ({ MAX_IMAGE_BYTES: 1_234 })); +vi.mock("@openclaw/media-core/constants", () => ({ + MAX_IMAGE_BYTES: 1_234, + mediaKindFromMime: (mime: string) => (mime.startsWith("image/") ? "image" : "unknown"), +})); vi.mock("../../image-sanitization.js", () => ({ resolveImageSanitizationLimits: hoisted.resolveImageSanitizationLimits, })); @@ -70,7 +73,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => { hoisted.resolveImageSanitizationLimits.mockReturnValue({ maxDimensionPx: 2048 }); hoisted.detectAndLoadPromptImages.mockResolvedValue({ images: [{ type: "image", data: "loaded", mimeType: "image/png" }], + imageFactIndexes: [null], detectedRefs: [], + failedMediaCount: 0, loadedCount: 1, skippedCount: 0, }); @@ -87,7 +92,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => { expect(second).toEqual({ images: [], + imageFactIndexes: [], detectedRefs: [], + failedMediaCount: 0, loadedCount: 0, skippedCount: 0, }); @@ -136,7 +143,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => { }); expect(result).toEqual({ images: [{ type: "image", data: "loaded", mimeType: "image/png" }], + imageFactIndexes: [null], detectedRefs: [], + failedMediaCount: 0, loadedCount: 1, skippedCount: 0, }); @@ -152,4 +161,81 @@ describe("prepareEmbeddedAttemptPromptExecution", () => { expect.objectContaining({ sandbox: undefined }), ); }); + + it("reports failed hydration without consuming the embedded attempt fact", async () => { + const base = createInput(); + const media = [{ path: "/tmp/missing.png", contentType: "image/png" }]; + const input = createInput({ attempt: { ...base.attempt, media } }); + hoisted.detectAndLoadPromptImages.mockResolvedValueOnce({ + images: [], + imageFactIndexes: [], + detectedRefs: [], + failedMediaCount: 1, + loadedCount: 0, + skippedCount: 1, + }); + + const result = await prepareEmbeddedAttemptPromptExecution(input); + + expect(result.failedMediaCount).toBe(1); + expect(input.attempt.media).toBe(media); + }); + + it("uses persisted facts and layout as the current-turn provenance authority", async () => { + const base = createInput(); + const persistedMessage = { + role: "user" as const, + content: "compare", + MediaPaths: ["/tmp/inline.png", "/tmp/offloaded.png"], + MediaTypes: ["image/png", "image/png"], + __openclaw: { + media: [ + { + path: "/tmp/inline.png", + contentType: "image/png", + hydrationSuppressed: true, + }, + { path: "/tmp/offloaded.png", contentType: "image/png" }, + ], + mediaImageLayout: { + slots: [ + { kind: "inline", factIndex: 0 }, + { kind: "offloaded", factIndex: 1 }, + ], + }, + }, + }; + const input = createInput({ + attempt: { + ...base.attempt, + media: [{ path: "/tmp/offloaded.png", contentType: "image/png" }], + userTurnTranscriptRecorder: { + message: persistedMessage, + resolveMessage: vi.fn(async () => persistedMessage), + } as unknown as NonNullable, + }, + }); + + await prepareEmbeddedAttemptPromptExecution(input); + + expect(hoisted.detectAndLoadPromptImages).toHaveBeenCalledWith( + expect.objectContaining({ + media: [ + expect.objectContaining({ + path: "/tmp/inline.png", + kind: "image", + hydrationSuppressed: true, + }), + expect.objectContaining({ path: "/tmp/offloaded.png", kind: "image" }), + ], + mediaImageLayout: { + slots: [ + { kind: "inline", factIndex: 0 }, + { kind: "offloaded", factIndex: 1 }, + ], + suppressedFactIndexes: [], + }, + }), + ); + }); }); diff --git a/src/agents/embedded-agent-runner/run/attempt-prompt-execution-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-prompt-execution-prepare.ts index 036c9a80efa6..980eb1a67b1b 100644 --- a/src/agents/embedded-agent-runner/run/attempt-prompt-execution-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-prompt-execution-prepare.ts @@ -1,6 +1,7 @@ /** Prepares prompt-lock ownership and prompt-local images for submission. */ import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants"; import type { OwnedSessionTranscriptCacheSnapshot } from "../../../config/sessions/transcript-write-context.js"; +import { resolveMediaFacts } from "../../../media/media-facts.js"; import { resolveImageSanitizationLimits } from "../../image-sanitization.js"; import type { SandboxContext } from "../../sandbox/types.js"; import type { AgentSession } from "../../sessions/index.js"; @@ -9,18 +10,31 @@ import { installPromptSubmissionLockRelease, } from "./attempt.session-lock.js"; import { detectAndLoadPromptImages } from "./images.js"; +import { + readPersistedMediaImageLayout, + readPersistedPromptMediaFacts, +} from "./prompt-image-metadata.js"; import type { EmbeddedRunAttemptParams } from "./types.js"; type PromptExecutionAttempt = Pick< EmbeddedRunAttemptParams, - "config" | "imageOrder" | "images" | "model" | "sessionFile" | "sessionKey" + | "config" + | "imageOrder" + | "images" + | "media" + | "model" + | "sessionFile" + | "sessionKey" + | "userTurnTranscriptRecorder" >; type PromptImageResult = Awaited>; function emptyPromptImages(): PromptImageResult { return { images: [], + imageFactIndexes: [], detectedRefs: [], + failedMediaCount: 0, loadedCount: 0, skippedCount: 0, }; @@ -57,12 +71,24 @@ export async function prepareEmbeddedAttemptPromptExecution(input: { input.sessionLockController.publishOwnedSessionFileSnapshot(snapshot), }); + const persistedMessage = + attempt.userTurnTranscriptRecorder?.message ?? + (await attempt.userTurnTranscriptRecorder?.resolveMessage()); + const persistedMedia = persistedMessage + ? (readPersistedPromptMediaFacts(persistedMessage) ?? + resolveMediaFacts(persistedMessage as unknown as Parameters[0])) + : []; + return await detectAndLoadPromptImages({ prompt: input.prompt, workspaceDir: input.effectiveWorkspace, model: attempt.model, existingImages: attempt.images, imageOrder: attempt.imageOrder, + media: persistedMedia.length > 0 ? persistedMedia : attempt.media, + mediaImageLayout: persistedMessage + ? readPersistedMediaImageLayout(persistedMessage) + : undefined, maxBytes: MAX_IMAGE_BYTES, maxDimensionPx: resolveImageSanitizationLimits(attempt.config).maxDimensionPx, workspaceOnly: input.effectiveFsWorkspaceOnly, diff --git a/src/agents/embedded-agent-runner/run/attempt-session-runtime-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-session-runtime-prepare.ts index dee0efb0e5f4..9d13ef13fd7f 100644 --- a/src/agents/embedded-agent-runner/run/attempt-session-runtime-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-session-runtime-prepare.ts @@ -39,6 +39,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: { activeContextEngine?: SessionManagerInput["activeContextEngine"]; agentDir: string; effectiveCwd: string; + effectiveFsWorkspaceOnly: boolean; effectiveWorkspace: string; initialSystemPrompt: string; isRawModelRun: boolean; @@ -167,6 +168,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: { attempt, computerContextEpoch: input.contextGuards.computerContextEpoch, effectiveCwd: input.effectiveCwd, + effectiveFsWorkspaceOnly: input.effectiveFsWorkspaceOnly, effectiveWorkspace: input.effectiveWorkspace, getPrePromptMessageCount: () => state.prePromptMessageCount, getPromptCache: () => state.promptCache, @@ -177,6 +179,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: { sessionAgentId: input.sessionManager.sessionAgentId, sessionManager, settingsManager, + sandbox: input.transport.sandbox, }); input.lifecycle.onContextGuardsInstalled(contextGuards.remove); diff --git a/src/agents/embedded-agent-runner/run/attempt-setup.ts b/src/agents/embedded-agent-runner/run/attempt-setup.ts index 1fc56353d21e..67302ebf001e 100644 --- a/src/agents/embedded-agent-runner/run/attempt-setup.ts +++ b/src/agents/embedded-agent-runner/run/attempt-setup.ts @@ -28,8 +28,59 @@ type PreparedProviderRuntimePluginHandle = ProviderRuntimePluginHandle & { prepared: true; }; -export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptParams) { +type AttemptWorkspaceParams = Pick< + EmbeddedRunAttemptParams, + | "agentId" + | "config" + | "cwd" + | "execOverrides" + | "sandboxSessionKey" + | "sessionId" + | "sessionKey" + | "workspaceDir" +>; + +/** Resolves the shared workspace and sandbox policy used by native and plugin harnesses. */ +export async function resolveAttemptWorkspaceSandbox(params: AttemptWorkspaceParams) { const resolvedWorkspace = resolveUserPath(params.workspaceDir); + await fs.mkdir(resolvedWorkspace, { recursive: true }); + const sandboxSessionKey = + params.sandboxSessionKey?.trim() || params.sessionKey?.trim() || params.sessionId; + const sandbox = await resolveSandboxContext({ + config: params.config, + execOverrides: params.execOverrides, + sessionKey: sandboxSessionKey, + workspaceDir: resolvedWorkspace, + }); + const effectiveWorkspace = + sandbox?.enabled && sandbox.workspaceAccess !== "rw" ? sandbox.workspaceDir : resolvedWorkspace; + const requestedCwd = params.cwd ? resolveUserPath(params.cwd) : undefined; + if (sandbox?.enabled && requestedCwd && requestedCwd !== resolvedWorkspace) { + throw new Error( + "cwd override is not supported for sandboxed embedded agent runs; omit cwd or use the agent workspace as cwd", + ); + } + await fs.mkdir(effectiveWorkspace, { recursive: true }); + const { sessionAgentId } = resolveSessionAgentIds({ + sessionKey: params.sessionKey, + config: params.config, + agentId: params.agentId, + }); + return { + effectiveCwd: sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace), + effectiveFsWorkspaceOnly: resolveAttemptFsWorkspaceOnly({ + config: params.config, + sessionAgentId, + }), + effectiveWorkspace, + resolvedWorkspace, + sandbox, + sandboxSessionKey, + sessionAgentId, + }; +} + +export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptParams) { // Ultra is a logical orchestration mode, not a provider effort. Preserve it for // prompt/status surfaces, then lower only at agent-core and provider boundaries. const agentCoreThinkingLevel = mapThinkingLevel(params.thinkLevel); @@ -73,28 +124,8 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara } }; - await fs.mkdir(resolvedWorkspace, { recursive: true }); - const sandboxSessionKey = - params.sandboxSessionKey?.trim() || params.sessionKey?.trim() || params.sessionId; - const sandbox = await resolveSandboxContext({ - config: params.config, - execOverrides: params.execOverrides, - sessionKey: sandboxSessionKey, - workspaceDir: resolvedWorkspace, - }); - const effectiveWorkspace = sandbox?.enabled - ? sandbox.workspaceAccess === "rw" - ? resolvedWorkspace - : sandbox.workspaceDir - : resolvedWorkspace; - const requestedCwd = params.cwd ? resolveUserPath(params.cwd) : undefined; - if (sandbox?.enabled && requestedCwd && requestedCwd !== resolvedWorkspace) { - throw new Error( - "cwd override is not supported for sandboxed embedded agent runs; omit cwd or use the agent workspace as cwd", - ); - } - const effectiveCwd = sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace); - await fs.mkdir(effectiveWorkspace, { recursive: true }); + const workspace = await resolveAttemptWorkspaceSandbox(params); + const { effectiveWorkspace } = workspace; const getCurrentAttemptPluginMetadataSnapshot = (): PluginMetadataSnapshot | undefined => params.preparedModelRuntime?.metadataSnapshot; @@ -141,22 +172,11 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara }; return providerRuntimeHandle; }; - const { sessionAgentId } = resolveSessionAgentIds({ - sessionKey: params.sessionKey, - config: params.config, - agentId: params.agentId, - }); - const effectiveFsWorkspaceOnly = resolveAttemptFsWorkspaceOnly({ - config: params.config, - sessionAgentId, - }); prepStages.mark("workspace-sandbox"); return { agentCoreThinkingLevel, - effectiveCwd, - effectiveFsWorkspaceOnly, - effectiveWorkspace, + ...workspace, emitCorePluginToolStageSummary, emitPrepStageSummary, getCurrentAttemptPluginMetadataSnapshot, @@ -164,9 +184,5 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara prepStages, proactiveSubagentOrchestration, providerThinkingLevel, - resolvedWorkspace, - sandbox, - sandboxSessionKey, - sessionAgentId, }; } diff --git a/src/agents/embedded-agent-runner/run/attempt.queue-message.test.ts b/src/agents/embedded-agent-runner/run/attempt.queue-message.test.ts index 358e895f3144..3964ed35c047 100644 --- a/src/agents/embedded-agent-runner/run/attempt.queue-message.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.queue-message.test.ts @@ -60,14 +60,18 @@ describe("embedded OpenClaw queued steering cancellation", () => { { path: "/tmp/a.png", contentType: "image/png" }, { path: "/tmp/b.pdf", contentType: "application/pdf" }, ]; + const imageOrder = ["offloaded", "inline"] as const; const activeSession: EmbeddedAgentActiveSessionSteerTarget = { steer, subscribe: () => () => {}, }; - await steerActiveSessionWithOptionalDeliveryWait(activeSession, "inspect both", { media }); + await steerActiveSessionWithOptionalDeliveryWait(activeSession, "inspect both", { + media, + imageOrder: [...imageOrder], + }); - expect(steer).toHaveBeenCalledWith("inspect both", undefined, undefined, media); + expect(steer).toHaveBeenCalledWith("inspect both", undefined, undefined, media, imageOrder); }); it("waits for the queued user message_end transcript boundary", async () => { diff --git a/src/agents/embedded-agent-runner/run/attempt.queue-message.ts b/src/agents/embedded-agent-runner/run/attempt.queue-message.ts index 8ccb4e4e3747..6c1667aba18b 100644 --- a/src/agents/embedded-agent-runner/run/attempt.queue-message.ts +++ b/src/agents/embedded-agent-runner/run/attempt.queue-message.ts @@ -4,6 +4,7 @@ import { toErrorObject } from "../../../infra/errors.js"; import type { ImageContent } from "../../../llm/types.js"; import type { MediaFact } from "../../../media/media-facts.js"; +import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js"; import type { UserTurnTranscriptRecorder } from "../../../sessions/user-turn-transcript.types.js"; import { cancelPendingAgentQuestionForSession, @@ -24,6 +25,7 @@ type EmbeddedAgentActiveSessionSteerTarget = { images?: ImageContent[], userTurnTranscriptRecorder?: UserTurnTranscriptRecorder, media?: MediaFact[], + imageOrder?: PromptImageOrderEntry[], ): Promise; subscribe(listener: (event: unknown) => void): () => void; }; @@ -37,9 +39,10 @@ function steerActiveSession( images?: ImageContent[], userTurnTranscriptRecorder?: UserTurnTranscriptRecorder, media?: MediaFact[], + imageOrder?: PromptImageOrderEntry[], ): Promise { if (media?.length) { - return activeSession.steer(text, images, userTurnTranscriptRecorder, media); + return activeSession.steer(text, images, userTurnTranscriptRecorder, media, imageOrder); } return userTurnTranscriptRecorder ? activeSession.steer(text, images, userTurnTranscriptRecorder) @@ -157,6 +160,7 @@ async function steerAndWaitForTranscriptCommit( userTurnTranscriptRecorder?: UserTurnTranscriptRecorder, images?: ImageContent[], media?: MediaFact[], + imageOrder?: PromptImageOrderEntry[], ): Promise { await new Promise((resolve, reject) => { let settled = false; @@ -243,6 +247,7 @@ async function steerAndWaitForTranscriptCommit( images, userTurnTranscriptRecorder, media, + imageOrder, ); steer.catch((err: unknown) => { finish(err); @@ -291,6 +296,7 @@ export async function steerActiveSessionWithOptionalDeliveryWait( options?.images, options?.userTurnTranscriptRecorder, options?.media, + options?.imageOrder, ); return; } @@ -301,5 +307,6 @@ export async function steerActiveSessionWithOptionalDeliveryWait( options.userTurnTranscriptRecorder, options.images, options.media, + options.imageOrder, ); } diff --git a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts index 16f9e1b308cd..25585cdb28a8 100644 --- a/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts +++ b/src/agents/embedded-agent-runner/run/attempt.spawn-workspace.test-support.ts @@ -216,7 +216,9 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => { const prepareSessionManagerForRunMock = vi.fn(async (_params?: unknown) => undefined); const detectAndLoadPromptImagesMock = vi.fn(async () => ({ images: [], + imageFactIndexes: [], detectedRefs: [], + failedMediaCount: 0, loadedCount: 0, skippedCount: 0, })); diff --git a/src/agents/embedded-agent-runner/run/attempt.ts b/src/agents/embedded-agent-runner/run/attempt.ts index 716e104c505c..e0686a1550eb 100644 --- a/src/agents/embedded-agent-runner/run/attempt.ts +++ b/src/agents/embedded-agent-runner/run/attempt.ts @@ -330,6 +330,7 @@ export async function runEmbeddedAttempt( ...(activeContextEngine ? { activeContextEngine } : {}), agentDir, effectiveCwd, + effectiveFsWorkspaceOnly, effectiveWorkspace, initialSystemPrompt: preparedSystemPrompt.systemPromptText, isRawModelRun, diff --git a/src/agents/embedded-agent-runner/run/history-image-prune.test.ts b/src/agents/embedded-agent-runner/run/history-image-prune.test.ts index 99f8c0cff25c..83b6b942e857 100644 --- a/src/agents/embedded-agent-runner/run/history-image-prune.test.ts +++ b/src/agents/embedded-agent-runner/run/history-image-prune.test.ts @@ -1,9 +1,18 @@ // History image prune tests keep provider replay compact by replacing stale -// image bytes and media references while preserving recent user context. +// image bytes and fact-owned projections while preserving recent user context. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; import type { ImageContent } from "openclaw/plugin-sdk/llm"; import { describe, expect, it } from "vitest"; +import { + attachRuntimePromptMediaFacts, + readRuntimePromptMediaFacts, +} from "../../../media/media-facts.js"; +import { buildPersistedUserTurnMessage } from "../../../sessions/user-turn-transcript.js"; import { castAgentMessage } from "../../test-helpers/agent-message-fixtures.js"; +import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js"; import { installHistoryImagePruneContextTransform, pruneProcessedHistoryImages, @@ -12,6 +21,8 @@ import { const PRUNED_HISTORY_IMAGE_MARKER = "[image data removed - already processed by model]"; const PRUNED_HISTORY_MEDIA_REFERENCE_MARKER = "[media reference removed - already processed by model]"; +const TINY_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg=="; function expectArrayMessageContent( message: AgentMessage | undefined, @@ -105,9 +116,23 @@ describe("pruneProcessedHistoryImages", () => { expect(content[0]?.type).toBe("text"); }); - it("scrubs old media attachment markers from text blocks", () => { - // Text references are scrubbed alongside image blocks so old paths and - // media URIs cannot rehydrate stale images on a later replay. + it("strips explicit-image provenance when its old image block is pruned", () => { + const message = castAgentMessage({ + role: "user", + content: [{ type: "text", text: "explicit image" }, { ...image }], + __openclaw: { + mediaImageBlockFactIndexes: [null], + mediaImageLayout: { slots: [{ kind: "inline" }] }, + }, + }); + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const meta = (pruned[0] as unknown as Record)["__openclaw"]; + + expect(meta).toBeUndefined(); + }); + + it("redacts factless legacy attachment text while pruning old image blocks", () => { const messages: AgentMessage[] = [ castAgentMessage({ role: "user", @@ -147,7 +172,7 @@ describe("pruneProcessedHistoryImages", () => { expectContentBlock(originalContent[1], { type: "image", data: "abc" }); }); - it("scrubs old media attachment markers from string content without image blocks", () => { + it("redacts factless legacy attachment text without image blocks", () => { const messages: AgentMessage[] = [ castAgentMessage({ role: "user", @@ -157,16 +182,11 @@ describe("pruneProcessedHistoryImages", () => { ]; const pruned = expectPrunedMessages(messages); - - const firstUser = pruned[0] as Extract | undefined; - expect(firstUser?.content).toBe(`please remember ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}`); - const originalUser = messages[0] as Extract | undefined; - expect(originalUser?.content).toBe( - "please remember [media attached: media://inbound/stale-image.png]", - ); + const user = pruned[0] as Extract | undefined; + expect(user?.content).toBe(`please remember ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}`); }); - it("prunes marked bare late-media turns identically to legacy literal turns", () => { + it("prunes fact-owned and factless legacy late-media projections", () => { const fields = { role: "user" as const, MediaPath: "media://inbound/stale-image.png", @@ -178,8 +198,8 @@ describe("pruneProcessedHistoryImages", () => { __openclaw: { lateMedia: true }, }); const legacyString = castAgentMessage({ - ...fields, content: "[media attached: media://inbound/stale-image.png]", + role: "user", }); const markedArray = castAgentMessage({ ...fields, @@ -187,7 +207,7 @@ describe("pruneProcessedHistoryImages", () => { __openclaw: { lateMedia: true }, }); const legacyArray = castAgentMessage({ - ...fields, + role: "user", content: [ { type: "text", text: "[media attached: media://inbound/stale-image.png]" }, { ...image }, @@ -199,12 +219,20 @@ describe("pruneProcessedHistoryImages", () => { const prunedMarkedArray = expectPrunedMessages([markedArray, ...oldEnoughTail()]); const prunedLegacyArray = expectPrunedMessages([legacyArray, ...oldEnoughTail()]); const markedStringOutput = prunedMarkedString as unknown as Array<{ content?: unknown }>; - const legacyStringOutput = prunedLegacyString as unknown as Array<{ content?: unknown }>; const markedArrayOutput = prunedMarkedArray as unknown as Array<{ content?: unknown }>; + const legacyStringOutput = prunedLegacyString as unknown as Array<{ content?: unknown }>; const legacyArrayOutput = prunedLegacyArray as unknown as Array<{ content?: unknown }>; - expect(markedStringOutput[0]?.content).toBe(legacyStringOutput[0]?.content); - expect(markedArrayOutput[0]?.content).toEqual(legacyArrayOutput[0]?.content); + expect(markedStringOutput[0]?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); + expect(legacyStringOutput[0]?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); + expect(markedArrayOutput[0]?.content).toEqual([ + { type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER }, + { type: "text", text: PRUNED_HISTORY_IMAGE_MARKER }, + ]); + expect(legacyArrayOutput[0]?.content).toEqual([ + { type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER }, + { type: "text", text: PRUNED_HISTORY_IMAGE_MARKER }, + ]); }); it("does not replace a distinct caption on an old marked late-media turn", () => { @@ -216,12 +244,178 @@ describe("pruneProcessedHistoryImages", () => { __openclaw: { lateMedia: true }, }); - expect(pruneProcessedHistoryImages([message, ...oldEnoughTail()])).toBeNull(); - const output = [message] as unknown as Array<{ content?: unknown }>; + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const output = pruned as unknown as Array<{ content?: unknown; MediaPath?: unknown }>; expect(output[0]?.content).toBe("resolved subtitle"); + expect(output[0]?.MediaPath).toBeUndefined(); }); - it("scrubs bare old inbound media URIs from tool results", () => { + it("drops runtime facts from captioned old turns without changing their text", () => { + const message = attachRuntimePromptMediaFacts( + castAgentMessage({ role: "user", content: "caption stays byte-identical" }), + [{ path: "/tmp/stale.png", contentType: "image/png" }], + ); + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe("caption stays byte-identical"); + expect(firstUser && readRuntimePromptMediaFacts(firstUser)).toBeUndefined(); + expect(readRuntimePromptMediaFacts(message)).toHaveLength(1); + }); + + it("redacts the exact fact-owned projection from a captioned old turn", () => { + const imagePath = "/tmp/stale-owned.png"; + const message = attachRuntimePromptMediaFacts( + castAgentMessage({ + role: "user", + content: `[media attached: ${imagePath} (image/png)]\n\ncaption stays`, + }), + [{ path: imagePath, contentType: "image/png" }], + ); + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe(`${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}\n\ncaption stays`); + }); + + it("redacts a persisted fact projection with a distinct URL", () => { + const imagePath = "/tmp/stale-owned.png"; + const imageUrl = "https://example.test/stale-owned.png"; + const message = castAgentMessage({ + role: "user", + content: `[media attached: ${imagePath} (image/png) | ${imageUrl}]`, + __openclaw: { + media: [ + { + path: imagePath, + url: imageUrl, + contentType: "image/png", + hydrationSuppressed: true, + }, + ], + }, + }); + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); + }); + + it("redacts a persisted relative projection after legacy path canonicalization", () => { + const message = buildPersistedUserTurnMessage({ + text: "[media attached: ./old.png (image/png)]", + media: [ + { + path: "./old.png", + workspaceDir: "/workspace", + contentType: "image/png", + }, + ], + }) as AgentMessage; + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); + }); + + it("redacts a persisted relative projection without a dot prefix", () => { + const message = buildPersistedUserTurnMessage({ + text: "[media attached: sub/old.png (image/png)]", + media: [ + { + path: "sub/old.png", + workspaceDir: "/workspace", + contentType: "image/png", + }, + ], + }) as AgentMessage; + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); + }); + + it("does not claim a basename-only marker for an absolute owned fact", () => { + const message = buildPersistedUserTurnMessage({ + text: "caption mentions [media attached: ./photo.png (image/png)]", + media: [{ path: "/workspace/sub/photo.png", contentType: "image/png" }], + }) as AgentMessage; + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe("caption mentions [media attached: ./photo.png (image/png)]"); + }); + + it("preserves an unrelated legacy-shaped marker in a fact-backed caption", () => { + const message = buildPersistedUserTurnMessage({ + text: "caption mentions [media attached: ./unrelated.png (image/png)]", + media: [{ path: "./owned.png", workspaceDir: "/workspace", contentType: "image/png" }], + }) as AgentMessage; + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe( + "caption mentions [media attached: ./unrelated.png (image/png)]", + ); + }); + + it("redacts separately projected fact lines in distinct text blocks", () => { + const firstPath = "/tmp/first-owned.png"; + const secondPath = "/tmp/second-owned.png"; + const message = attachRuntimePromptMediaFacts( + castAgentMessage({ + role: "user", + content: [ + { type: "text", text: `[media attached: ${firstPath} (image/png)]` }, + { type: "text", text: `[media attached: ${secondPath} (image/png)]` }, + ], + }), + [ + { path: firstPath, contentType: "image/png" }, + { path: secondPath, contentType: "image/png" }, + ], + ); + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + expect(expectArrayMessageContent(pruned[0], "expected redacted text blocks")).toEqual([ + { type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER }, + { type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER }, + ]); + }); + + it("redacts a fact-backed legacy image-source projection", () => { + const imagePath = "/tmp/legacy-owned.jpg"; + const message = castAgentMessage({ + role: "user", + content: `[Image: source: ${imagePath}]\ncaption stays`, + MediaPath: imagePath, + MediaPaths: [imagePath], + MediaType: "image/jpeg", + MediaTypes: ["image/jpeg"], + }); + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe(`${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}\ncaption stays`); + }); + + it("preserves a bare claim-check URI in a fact-backed caption", () => { + const mediaRef = "media://inbound/captioned.png"; + const message = castAgentMessage({ + role: "user", + content: `caption mentions ${mediaRef} as text`, + MediaPath: mediaRef, + MediaPaths: [mediaRef], + MediaType: "image/png", + MediaTypes: ["image/png"], + }); + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const firstUser = pruned[0] as Extract | undefined; + expect(firstUser?.content).toBe(`caption mentions ${mediaRef} as text`); + }); + + it("redacts bare old inbound media URIs from factless tool results", () => { const messages: AgentMessage[] = [ castAgentMessage({ role: "toolResult", @@ -232,15 +426,8 @@ describe("pruneProcessedHistoryImages", () => { ]; const pruned = expectPrunedMessages(messages); - const toolResult = pruned[0] as Extract | undefined; expect(toolResult?.content).toBe(`previous ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER} result`); - const originalToolResult = messages[0] as - | Extract - | undefined; - expect(originalToolResult?.content).toBe( - "previous media://inbound/stale-screenshot.png result", - ); }); it("keeps image blocks that belong to the third-most-recent assistant turn", () => { @@ -453,4 +640,133 @@ describe("installHistoryImagePruneContextTransform", () => { restore(); expect(agent.transformContext).toBe(originalTransformContext); }); + + it("does not legacy-redact a fact-backed caption on the wrapper second pass", async () => { + const mediaRef = "media://inbound/captioned.png"; + const messages: AgentMessage[] = [ + castAgentMessage({ + role: "user", + content: `caption mentions ${mediaRef} as text`, + media: [{ url: mediaRef, contentType: "image/png" }], + }), + ...oldEnoughTail(), + ]; + const agent = { + transformContext: async (input: AgentMessage[]) => + input.map((message) => ({ ...message })) as AgentMessage[], + }; + const restore = installHistoryImagePruneContextTransform(agent); + + const replay = await agent.transformContext(messages); + + expect((replay[0] as { content?: unknown }).content).toBe( + `caption mentions ${mediaRef} as text`, + ); + restore(); + }); + + it("hydrates recent facts before an existing transform clones messages", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-history-hydrate-")); + const imagePath = path.join(workspaceDir, "photo.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const message = attachRuntimePromptMediaFacts( + castAgentMessage({ role: "user", content: [{ type: "text", text: "describe" }] }), + [{ path: imagePath, contentType: "image/png" }], + ); + const agent = { + transformContext: async (messages: AgentMessage[]) => + messages.map((entry) => { + if (!("content" in entry)) { + return { ...entry }; + } + return { + ...entry, + content: Array.isArray(entry.content) + ? entry.content.map((block: (typeof entry.content)[number]) => ({ ...block })) + : entry.content, + }; + }) as AgentMessage[], + }; + const restore = installHistoryImagePruneContextTransform(agent, { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + + try { + const replay = await agent.transformContext([message]); + expect(expectArrayMessageContent(replay[0], "expected hydrated content")).toEqual([ + { type: "text", text: "describe" }, + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + ]); + } finally { + restore(); + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("strips nested media metadata before old turns can rehydrate", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-pruned-nested-media-")); + const imagePath = path.join(workspaceDir, "old.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const baseBridge = createHostSandboxFsBridge(workspaceDir); + let hydrationReadCount = 0; + const bridge = { + ...baseBridge, + readFile: async (params: Parameters[0]) => { + hydrationReadCount++; + return await baseBridge.readFile(params); + }, + }; + const message = castAgentMessage({ + role: "user", + content: "[media attached: ./old.png (image/png)]", + __openclaw: { + media: [{ path: "./old.png", contentType: "image/png" }], + mediaImageBlockFactIndexes: [0], + mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] }, + }, + }); + const agent: { + transformContext?: (messages: AgentMessage[]) => Promise | AgentMessage[]; + } = {}; + const restore = installHistoryImagePruneContextTransform(agent, { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + sandbox: { root: workspaceDir, bridge }, + }); + + try { + const replay = await agent.transformContext?.([message, ...oldEnoughTail()]); + const meta = (replay?.[0] as unknown as Record)?.["__openclaw"] as + | Record + | undefined; + expect(hydrationReadCount).toBe(0); + expect(meta?.media).toBeUndefined(); + expect(meta?.mediaImageBlockFactIndexes).toBeUndefined(); + expect(meta?.mediaImageLayout).toBeUndefined(); + } finally { + restore(); + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("treats identity-less nested facts as structured ownership", () => { + const message = castAgentMessage({ + role: "user", + content: "[media attached: /tmp/unknown.png (image/png)]", + __openclaw: { + media: [{ kind: "image" }], + mediaImageLayout: { slots: [], suppressedFactIndexes: [0] }, + }, + }); + + const pruned = expectPrunedMessages([message, ...oldEnoughTail()]); + const first = pruned[0] as unknown as Record; + const meta = first["__openclaw"] as Record | undefined; + expect(first.content).toBe("[media attached: /tmp/unknown.png (image/png)]"); + expect(meta?.media).toBeUndefined(); + expect(meta?.mediaImageLayout).toBeUndefined(); + }); }); diff --git a/src/agents/embedded-agent-runner/run/history-image-prune.ts b/src/agents/embedded-agent-runner/run/history-image-prune.ts index 6699928d6855..3f6c14d13636 100644 --- a/src/agents/embedded-agent-runner/run/history-image-prune.ts +++ b/src/agents/embedded-agent-runner/run/history-image-prune.ts @@ -1,20 +1,31 @@ +import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js"; +import { + normalizeMediaFacts, + projectMediaFacts, + readRuntimePromptMediaFacts, + resolveMediaFacts, + type MediaFact, +} from "../../../media/media-facts.js"; /** * Prunes already-processed image payloads from replayed prompt history. */ import { buildLateMediaAttachedProjection } from "../../../sessions/user-turn-transcript.js"; import type { AgentMessage } from "../../runtime/index.js"; import { hasNonBlankUserText } from "./attempt.user-message-boundary.js"; +import { hydratePromptMediaMessages } from "./images.js"; /** Replacement text for old image blocks that were already available to the model. */ const PRUNED_HISTORY_IMAGE_MARKER = "[image data removed - already processed by model]"; -/** Replacement text for old textual media references that would otherwise be reloaded. */ +/** Replacement text for fact-owned late-media projections already processed by the model. */ const PRUNED_HISTORY_MEDIA_REFERENCE_MARKER = "[media reference removed - already processed by model]"; -const MEDIA_ATTACHED_HISTORY_REF_PATTERN = /\[media attached(?:\s+\d+\/\d+)?:\s*[^\]]+\]/gi; -const MESSAGE_IMAGE_HISTORY_REF_PATTERN = /\[Image:\s*source:\s*[^\]]+\]/gi; -const INBOUND_MEDIA_URI_HISTORY_REF_PATTERN = /\bmedia:\/\/inbound\/[^\]\s/\\]+/g; +// Legacy replay hygiene only: factless pre-MediaFact rows past the prune cutoff +// retain no attachment ownership. Fact-bearing messages never use these patterns. +const LEGACY_MEDIA_ATTACHED_PATTERN = /\[media attached(?:\s+\d+\/\d+)?:\s*[^\]]+\]/gi; +const LEGACY_IMAGE_SOURCE_PATTERN = /\[Image:\s*source:\s*[^\]]+\]/gi; +const LEGACY_INBOUND_MEDIA_URI_PATTERN = /\bmedia:\/\/inbound\/[^\]\s/\\]+/g; type PrunableContextAgent = { transformContext?: ( @@ -29,6 +40,17 @@ type PrunableContextAgent = { * ones, so text-only turns consume the window. */ const PRESERVE_RECENT_COMPLETED_TURNS = 3; +const PERSISTED_MEDIA_FIELD_KEYS = [ + "media", + "MediaPath", + "MediaPaths", + "MediaUrl", + "MediaUrls", + "MediaType", + "MediaTypes", + "MediaTranscribedIndexes", + "MediaWorkspaceDir", +] as const; function resolvePruneBeforeIndex(messages: AgentMessage[]): number { const completedTurnStarts: number[] = []; @@ -66,18 +88,133 @@ function resolvePruneBeforeIndex(messages: AgentMessage[]): number { return completedTurnStarts.at(-PRESERVE_RECENT_COMPLETED_TURNS) ?? -1; } -function pruneHistoryMediaReferenceText(text: string): string { +function resolveMessageMediaFacts(message: AgentMessage): MediaFact[] { + const runtimeMedia = readRuntimePromptMediaFacts(message); + if (runtimeMedia) { + return runtimeMedia; + } + const meta = (message as unknown as Record)["__openclaw"]; + const nestedMedia = + meta && typeof meta === "object" && !Array.isArray(meta) + ? (meta as Record).media + : undefined; + return Array.isArray(nestedMedia) + ? normalizeMediaFacts(nestedMedia as MediaFact[]) + : resolveMediaFacts(message as unknown as Parameters[0]); +} + +function wasStructurallyMediaPruned(message: AgentMessage): boolean { + const meta = (message as unknown as Record)["__openclaw"]; + return ( + Boolean(meta) && + typeof meta === "object" && + !Array.isArray(meta) && + (meta as Record).mediaImagePruned === true + ); +} + +function replaceLegacyFactlessMediaText(text: string): string { return text - .replace(MEDIA_ATTACHED_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER) - .replace(MESSAGE_IMAGE_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER) - .replace(INBOUND_MEDIA_URI_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); + .replace(LEGACY_MEDIA_ATTACHED_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER) + .replace(LEGACY_IMAGE_SOURCE_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER) + .replace(LEGACY_INBOUND_MEDIA_URI_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); +} + +function factOwnsMarkerIdentity(identity: string, media: MediaFact[]): boolean { + const normalizedIdentity = identity.replaceAll("\\", "/"); + return media.some((fact) => + [fact.path, fact.url].some((alias) => alias?.replaceAll("\\", "/") === normalizedIdentity), + ); +} + +function extractMediaAttachedIdentity(marker: string): string { + const content = marker.replace(/^\[media attached(?:\s+\d+\/\d+)?:\s*/i, "").slice(0, -1); + const mimeIndex = content.lastIndexOf(" ("); + const urlIndex = content.indexOf(" | "); + const endIndexes = [mimeIndex, urlIndex].filter((index) => index >= 0); + const endIndex = endIndexes.length > 0 ? Math.min(...endIndexes) : content.length; + return content.slice(0, endIndex).trim(); +} + +function replaceOwnedLegacyMediaMarkers(text: string, media: MediaFact[]): string { + return text + .replace(LEGACY_MEDIA_ATTACHED_PATTERN, (marker) => + factOwnsMarkerIdentity(extractMediaAttachedIdentity(marker), media) + ? PRUNED_HISTORY_MEDIA_REFERENCE_MARKER + : marker, + ) + .replace(LEGACY_IMAGE_SOURCE_PATTERN, (marker) => { + const identity = marker + .replace(/^\[Image:\s*source:\s*/i, "") + .slice(0, -1) + .trim(); + return factOwnsMarkerIdentity(identity, media) + ? PRUNED_HISTORY_MEDIA_REFERENCE_MARKER + : marker; + }); +} + +function replaceOwnedMediaProjection(text: string, media: MediaFact[]): string { + if (media.length === 0) { + return text; + } + const projectionLines = new Set(); + for (const facts of [media, ...media.map((fact) => [fact])]) { + const projection = buildInboundMediaNoteProjection({ + media: facts, + ...projectMediaFacts(facts, "channel"), + }).text; + for (const line of projection?.split("\n") ?? []) { + if (line) { + projectionLines.add(line); + } + } + } + let redacted = text; + for (const line of projectionLines) { + redacted = redacted.replaceAll(line, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); + } + for (const fact of media) { + for (const alias of [fact.path, fact.url].filter((value): value is string => Boolean(value))) { + redacted = redacted + .replaceAll(`[Image: source: ${alias}]`, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER) + .replaceAll(`[media attached: ${alias}]`, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); + } + } + return replaceOwnedLegacyMediaMarkers(redacted, media); } function cloneMessageWithContent( message: Extract, content: typeof message.content, + dropMedia = false, + dropImageMetadata = dropMedia, ): AgentMessage { - return { ...message, content } as AgentMessage; + const clone = { ...message, content } as AgentMessage & Record; + if (dropMedia) { + for (const key of PERSISTED_MEDIA_FIELD_KEYS) { + delete clone[key]; + } + } + if (dropImageMetadata) { + const meta = clone["__openclaw"]; + const nextMeta = + meta && typeof meta === "object" && !Array.isArray(meta) + ? { ...(meta as Record) } + : {}; + delete nextMeta.mediaImageBlockFactIndexes; + delete nextMeta.mediaImageLayout; + if (dropMedia) { + delete nextMeta.media; + nextMeta.mediaImagePruned = true; + } + if (Object.keys(nextMeta).length > 0) { + clone["__openclaw"] = nextMeta; + } else { + delete clone["__openclaw"]; + } + } + return clone; } /** Prunes old image payloads and references before later LLM-boundary synthesis. */ @@ -93,12 +230,18 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess if (!message || (message.role !== "user" && message.role !== "toolResult")) { continue; } + const media = message.role === "user" ? resolveMessageMediaFacts(message) : []; + const hasOwnedMedia = media.length > 0; + const structuredMediaWasPruned = wasStructurallyMediaPruned(message); // Materialize blank marked turns here so this earlier boundary still prunes stale paths. - const lateMediaText = + const lateMediaProjection = message.role === "user" && !hasNonBlankUserText(message.content) - ? buildLateMediaAttachedProjection(message).text + ? buildLateMediaAttachedProjection(message) : undefined; + const lateMediaText = lateMediaProjection?.media + .map(() => PRUNED_HISTORY_MEDIA_REFERENCE_MARKER) + .join("\n"); const content = lateMediaText ? Array.isArray(message.content) ? ([{ type: "text", text: lateMediaText }, ...message.content] as typeof message.content) @@ -106,10 +249,14 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess : message.content; if (typeof content === "string") { - const prunedText = pruneHistoryMediaReferenceText(content); - if (prunedText !== message.content) { + const nextText = hasOwnedMedia + ? replaceOwnedMediaProjection(content, media) + : structuredMediaWasPruned + ? content + : replaceLegacyFactlessMediaText(content); + if (nextText !== message.content || hasOwnedMedia) { prunedMessages ??= messages.slice(); - prunedMessages[i] = cloneMessageWithContent(message, prunedText); + prunedMessages[i] = cloneMessageWithContent(message, nextText, hasOwnedMedia); } continue; } @@ -121,16 +268,34 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess const nextContent = content.map((block) => { const typed = block as { type?: unknown; text?: unknown } | null | undefined; if (typed?.type === "text" && typeof typed.text === "string") { - const text = pruneHistoryMediaReferenceText(typed.text); - return text === typed.text ? block : ({ ...block, text } as (typeof content)[number]); + const text = hasOwnedMedia + ? replaceOwnedMediaProjection(typed.text, media) + : structuredMediaWasPruned + ? typed.text + : replaceLegacyFactlessMediaText(typed.text); + if (text !== typed.text) { + return { ...block, text } as (typeof content)[number]; + } } return typed?.type === "image" ? ({ type: "text", text: PRUNED_HISTORY_IMAGE_MARKER } as (typeof content)[number]) : block; }); - if (lateMediaText || nextContent.some((block, index) => block !== content[index])) { + const prunedImageBlock = content.some( + (block) => (block as { type?: unknown } | null | undefined)?.type === "image", + ); + if ( + hasOwnedMedia || + lateMediaText || + nextContent.some((block, index) => block !== content[index]) + ) { prunedMessages ??= messages.slice(); - prunedMessages[i] = cloneMessageWithContent(message, nextContent); + prunedMessages[i] = cloneMessageWithContent( + message, + nextContent, + hasOwnedMedia, + hasOwnedMedia || prunedImageBlock, + ); } } @@ -138,13 +303,20 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess } /** Installs an agent context transform that prunes old image/media history before model input. */ -export function installHistoryImagePruneContextTransform(agent: PrunableContextAgent): () => void { +export function installHistoryImagePruneContextTransform( + agent: PrunableContextAgent, + mediaOptions?: Parameters[1], +): () => void { const originalTransformContext = agent.transformContext; agent.transformContext = async (messages: AgentMessage[], signal?: AbortSignal) => { + const prunedInput = pruneProcessedHistoryImages(messages) ?? messages; + const hydratedInput = mediaOptions + ? await hydratePromptMediaMessages(prunedInput, mediaOptions) + : prunedInput; const transformed = originalTransformContext - ? await originalTransformContext.call(agent, messages, signal) - : messages; - const sourceMessages = Array.isArray(transformed) ? transformed : messages; + ? await originalTransformContext.call(agent, hydratedInput, signal) + : hydratedInput; + const sourceMessages = Array.isArray(transformed) ? transformed : hydratedInput; return pruneProcessedHistoryImages(sourceMessages) ?? sourceMessages; }; return () => { diff --git a/src/agents/embedded-agent-runner/run/images.combinations.test.ts b/src/agents/embedded-agent-runner/run/images.combinations.test.ts new file mode 100644 index 000000000000..056172938ebd --- /dev/null +++ b/src/agents/embedded-agent-runner/run/images.combinations.test.ts @@ -0,0 +1,157 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { createSolidPngBuffer } from "../../../../test/helpers/image-fixtures.js"; +import { detectAndLoadPromptImages } from "./images.js"; + +const HYDRATION_PARTS = ["inline", "offloaded", "suppressed", "explicit", "legacy"] as const; +type HydrationPart = (typeof HYDRATION_PARTS)[number]; +type HydrationCombination = { name: string; parts: readonly HydrationPart[] }; + +function combinations(values: readonly T[], size: number): T[][] { + if (size === 0) { + return [[]]; + } + return values.flatMap((value, index) => + combinations(values.slice(index + 1), size - 1).map((tail) => [value].concat(tail)), + ); +} + +const HYDRATION_COMBINATIONS: HydrationCombination[] = [2, 3].flatMap((size) => + combinations(HYDRATION_PARTS, size).map((parts) => ({ + name: parts.join(" + "), + parts, + })), +); + +describe("hydration combination matrix", () => { + it("attributes a suppressed-plus-inline sanitization failure to the inline fact", async () => { + const result = await detectAndLoadPromptImages({ + prompt: "already described", + media: [ + { + path: "/tmp/described-missing.png", + contentType: "image/png", + hydrationSuppressed: true, + }, + { path: "/tmp/inline.png", contentType: "image/png" }, + ], + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }], + imageOrder: ["inline"], + }); + + expect(result.images).toEqual([]); + expect(result.imageFactIndexes).toEqual([]); + expect(result.loadedCount).toBe(0); + expect(result.failedMediaCount).toBe(1); + }); + + it.each(HYDRATION_COMBINATIONS)( + "$name preserves materialization, order, and suppression invariants", + async (testCase: HydrationCombination) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-hydration-matrix-")); + const has = (part: HydrationPart) => testCase.parts.includes(part); + const inlineBuffer = createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 }); + const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 }); + const explicitBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 }); + const inlinePath = path.join(root, "inline.png"); + const offloadedPath = path.join(root, "offloaded.png"); + const explicitPath = path.join(root, "explicit.png"); + const suppressedPath = path.join(root, "suppressed-missing.png"); + if (has("offloaded")) { + await fs.writeFile(offloadedPath, offloadedBuffer); + } + if (has("explicit")) { + await fs.writeFile(explicitPath, explicitBuffer); + } + + const media: Array<{ + path: string; + contentType: string; + hydrationSuppressed?: boolean; + }> = []; + if (has("suppressed")) { + media.push({ + path: suppressedPath, + contentType: "image/png", + hydrationSuppressed: true, + }); + } + const inlineFactIndex = has("inline") ? media.length : undefined; + if (has("inline")) { + media.push({ path: inlinePath, contentType: "image/png" }); + } + const offloadedFactIndex = has("offloaded") ? media.length : undefined; + if (has("offloaded")) { + media.push({ path: offloadedPath, contentType: "image/png" }); + } + + const inlineImage = { + type: "image" as const, + data: inlineBuffer.toString("base64"), + mimeType: "image/png", + }; + const imageOrder = [ + ...(has("inline") ? (["inline"] as const) : []), + ...(has("offloaded") ? (["offloaded"] as const) : []), + ]; + const existingImages = has("inline") ? [inlineImage] : undefined; + const existingImageFactIndexes = + has("legacy") && inlineFactIndex !== undefined ? [inlineFactIndex] : undefined; + + try { + const result = await detectAndLoadPromptImages({ + prompt: has("explicit") ? `inspect ${explicitPath}` : "inspect attachments", + media, + workspaceDir: root, + model: { input: ["text", "image"] }, + existingImages, + existingImageFactIndexes, + imageOrder: has("legacy") ? undefined : imageOrder, + workspaceOnly: true, + }); + + const expectedImages = [ + ...(has("inline") ? [inlineImage] : []), + ...(has("offloaded") + ? [ + { + type: "image" as const, + data: offloadedBuffer.toString("base64"), + mimeType: "image/png", + }, + ] + : []), + ...(has("explicit") + ? [ + { + type: "image" as const, + data: explicitBuffer.toString("base64"), + mimeType: "image/png", + }, + ] + : []), + ]; + const expectedFactIndexes = [ + ...(inlineFactIndex === undefined ? [] : [inlineFactIndex]), + ...(offloadedFactIndex === undefined ? [] : [offloadedFactIndex]), + ...(has("explicit") ? [null] : []), + ]; + const expectedLoadedCount = Number(has("offloaded")) + Number(has("explicit")); + + expect(result.images).toEqual(expectedImages); + expect(result.imageFactIndexes).toEqual(expectedFactIndexes); + expect(result.loadedCount).toBe(expectedLoadedCount); + expect(result.failedMediaCount).toBe(0); + expect(result.images).toHaveLength( + Number(has("inline")) + Number(has("offloaded")) + Number(has("explicit")), + ); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/src/agents/embedded-agent-runner/run/images.media-refs.test.ts b/src/agents/embedded-agent-runner/run/images.media-refs.test.ts new file mode 100644 index 000000000000..1d79d46300c1 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/images.media-refs.test.ts @@ -0,0 +1,293 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { describe, expect, it } from "vitest"; +import { normalizeMediaFacts, resolveMediaFacts } from "../../../media/media-facts.js"; +import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js"; +import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js"; +import { detectAndLoadPromptImages, hasHydratableMediaImages } from "./images.js"; + +const TINY_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg=="; + +describe("fact-carried image references", () => { + it("counts only facts that will hydrate an image attachment", () => { + expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", kind: "image" }])).toBe(true); + // Legacy transcript projections persist bare kinds as the media type. + expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", contentType: "image" }])).toBe(true); + expect(hasHydratableMediaImages([{ path: "/tmp/anim.webp", contentType: "sticker" }])).toBe( + true, + ); + expect(hasHydratableMediaImages([{ kind: "image" }])).toBe(false); + expect( + hasHydratableMediaImages([{ url: "https://example.test/remote.png", kind: "image" }]), + ).toBe(false); + expect( + hasHydratableMediaImages([{ path: "https://example.test/remote.png", kind: "image" }]), + ).toBe(false); + expect(hasHydratableMediaImages([])).toBe(false); + expect(hasHydratableMediaImages(undefined)).toBe(false); + for (const contentType of ["audio", "video", "document"]) { + expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", contentType }])).toBe(false); + } + }); + + it("retains described-image suppression across fact copy boundaries", async () => { + const normalized = normalizeMediaFacts([ + { + path: "/tmp/described.png", + contentType: "image/png", + hydrationSuppressed: true, + }, + ]); + const reprojected = resolveMediaFacts({ + media: normalized, + MediaPaths: ["/tmp/stale.png"], + MediaTypes: ["application/octet-stream"], + }); + + expect(reprojected).toEqual([ + expect.objectContaining({ + path: "/tmp/described.png", + contentType: "image/png", + kind: "image", + hydrationSuppressed: true, + }), + ]); + const result = await detectAndLoadPromptImages({ + prompt: "already described", + media: reprojected, + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + }); + expect(result.failedMediaCount).toBe(0); + expect(result.images).toEqual([]); + }); + + it("pairs identity-less facts with existing inline images when order metadata is absent", async () => { + const existingImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const result = await detectAndLoadPromptImages({ + prompt: "look", + media: [{ kind: "image", contentType: "image/png" }], + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + existingImages: [existingImage], + }); + + expect(result.failedMediaCount).toBe(0); + expect(result.images).toEqual([existingImage]); + }); + + it("loads an explicit ref matching a fact sliced into an inline slot", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-inline-explicit-ref-")); + const imagePath = path.join(workspaceDir, "photo.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + + try { + const result = await detectAndLoadPromptImages({ + prompt: `compare ${imagePath}`, + media: [{ path: imagePath, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + existingImages: [inlineImage], + imageOrder: ["inline"], + workspaceOnly: true, + }); + + expect(result.loadedCount).toBe(1); + expect(result.failedMediaCount).toBe(0); + expect(result.images).toEqual([ + inlineImage, + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("keeps identity-bearing refs when image order metadata has more inline slots", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-inline-order-")); + const imagePath = path.join(workspaceDir, "offloaded.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const firstInline = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const secondInline = { ...firstInline }; + + try { + const result = await detectAndLoadPromptImages({ + prompt: "compare", + media: [{ path: imagePath, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + existingImages: [firstInline, secondInline], + imageOrder: ["inline", "inline"], + workspaceOnly: true, + }); + + expect(result.loadedCount).toBe(1); + expect(result.failedMediaCount).toBe(0); + expect(result.images).toEqual([ + firstInline, + secondInline, + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + ]); + expect(result.imageFactIndexes).toEqual([null, null, 0]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("fails an exact inline slot whose image block is missing", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-missing-inline-")); + const imagePath = path.join(workspaceDir, "stale.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + + try { + const result = await detectAndLoadPromptImages({ + prompt: "inspect", + media: [{ path: imagePath, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + imageOrder: ["inline"], + workspaceOnly: true, + }); + + expect(result.loadedCount).toBe(0); + expect(result.failedMediaCount).toBe(1); + expect(result.images).toEqual([]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("hydrates a fact whose only local identity is a file URL", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-file-url-")); + const imagePath = path.join(workspaceDir, "photo.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + + try { + const result = await detectAndLoadPromptImages({ + prompt: "", + media: [{ path: pathToFileURL(imagePath).href, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect(result.images).toEqual([ + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("hydrates managed inbound media URIs before workspace path resolution", async () => { + // Managed media URIs are canonical inbound attachment handles and should + // work even when workspaceOnly would reject ordinary outside paths. + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-uri-")); + const workspaceDir = path.join(stateDir, "workspace-agent"); + const inboundDir = path.join(stateDir, "media", "inbound"); + const mediaId = "telegram-photo.png"; + await fs.mkdir(workspaceDir, { recursive: true }); + await fs.mkdir(inboundDir, { recursive: true }); + await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64")); + const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + + try { + const result = await detectAndLoadPromptImages({ + prompt: "", + media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + const image = result.images[0]; + + expect(image?.type).toBe("image"); + expect(image?.mimeType).toBe("image/png"); + expect(image?.data).toBe(TINY_PNG_BASE64); + } finally { + envSnapshot.restore(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + + it("hydrates sandbox-staged inbound media URIs", async () => { + const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-sbx-uri-")); + const inboundDir = path.join(sandboxRoot, "media", "inbound"); + const mediaId = "telegram-photo.png"; + await fs.mkdir(inboundDir, { recursive: true }); + await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64")); + + try { + const result = await detectAndLoadPromptImages({ + prompt: "", + media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }], + model: { input: ["text", "image"] }, + workspaceDir: sandboxRoot, + workspaceOnly: true, + sandbox: { + root: sandboxRoot, + bridge: createHostSandboxFsBridge(sandboxRoot), + }, + }); + const image = result.images[0]; + + expect(image?.type).toBe("image"); + expect(image?.mimeType).toBe("image/png"); + expect(image?.data).toBe(TINY_PNG_BASE64); + } finally { + await fs.rm(sandboxRoot, { recursive: true, force: true }); + } + }); + + it.each([ + ["traversal", "media://inbound/../secret.png"], + ["encoded traversal", "media://inbound/%2e%2e%2fsecret.png"], + ["null byte", "media://inbound/secret%00.png"], + ])("rejects %s claim-check facts", async (_label: string, mediaUrl: string) => { + const result = await detectAndLoadPromptImages({ + prompt: "legacy ticket is carried structurally", + media: [{ url: mediaUrl, contentType: "image/png" }], + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + }); + + expect(result.loadedCount).toBe(0); + expect(result.skippedCount).toBe(1); + expect(result.images).toHaveLength(0); + }); + + it("allows sandbox-validated host paths outside default media roots", async () => { + const homeDir = os.homedir(); + await fs.mkdir(homeDir, { recursive: true }); + const sandboxParent = await fs.mkdtemp(path.join(homeDir, "openclaw-sandbox-image-")); + try { + const sandboxRoot = path.join(sandboxParent, "sandbox"); + await fs.mkdir(sandboxRoot, { recursive: true }); + const imagePath = path.join(sandboxRoot, "photo.png"); + const pngB64 = TINY_PNG_BASE64; + await fs.writeFile(imagePath, Buffer.from(pngB64, "base64")); + + const result = await detectAndLoadPromptImages({ + prompt: "", + media: [{ path: "./photo.png", contentType: "image/png" }], + model: { input: ["text", "image"] }, + workspaceDir: sandboxRoot, + sandbox: { + root: sandboxRoot, + bridge: createHostSandboxFsBridge(sandboxRoot), + }, + }); + const image = result.images[0]; + + expect(image?.type).toBe("image"); + expect(image?.mimeType).toBe("image/png"); + expect(image?.data).toBe(TINY_PNG_BASE64); + } finally { + await fs.rm(sandboxParent, { recursive: true, force: true }); + } + }); +}); diff --git a/src/agents/embedded-agent-runner/run/images.media-refs.ts b/src/agents/embedded-agent-runner/run/images.media-refs.ts new file mode 100644 index 000000000000..2f8a377c26a9 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/images.media-refs.ts @@ -0,0 +1,160 @@ +import { safeFileURLToPath } from "../../../infra/local-file-access.js"; +import { + isImageMediaFact, + normalizeMediaFacts, + type MediaFact, +} from "../../../media/media-facts.js"; +import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js"; +import { resolveUserPath } from "../../../utils.js"; + +const URL_SCHEME_PATTERN = /^[a-z][a-z0-9+.-]*:/i; +const WINDOWS_DRIVE_PATH_PATTERN = /^[A-Za-z]:[\\/]/; + +type DetectedImageRef = { + raw: string; + type: "path" | "media-uri"; + resolved: string; +}; + +export type MediaImageRef = DetectedImageRef & { + aliases: string[]; + detect?: boolean; + factIndex: number; + hydrate: boolean; + workspaceDir?: string; +}; + +export function isOpenClawCliImageCachePath(filePath: string): boolean { + const parts = filePath.replaceAll("\\", "/").split("/"); + return parts.some((part, index) => { + if (part === ".openclaw-cli-images") { + return true; + } + const parent = parts[index - 1] ?? ""; + return part === "openclaw-cli-images" && /^openclaw(?:-\d+)?$/.test(parent); + }); +} + +function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef | undefined { + if (!isImageMediaFact(fact)) { + return undefined; + } + const mediaUri = [fact.url, fact.path].find((value) => value?.startsWith("media://inbound/")); + const identity = mediaUri ?? fact.path ?? fact.url; + if (!identity) { + return fact.hydrationSuppressed === true + ? { + aliases: [], + detect: false, + factIndex, + raw: "", + type: "path", + resolved: "", + hydrate: false, + ...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}), + } + : undefined; + } + let resolved = mediaUri; + if (!resolved && identity && /^file:/i.test(identity)) { + try { + resolved = safeFileURLToPath(identity); + } catch { + resolved = undefined; + } + } else if ( + !resolved && + identity && + (!URL_SCHEME_PATTERN.test(identity) || WINDOWS_DRIVE_PATH_PATTERN.test(identity)) + ) { + resolved = identity; + } + if (resolved?.startsWith("~")) { + resolved = resolveUserPath(resolved); + } + const hydrate = fact.hydrationSuppressed !== true; + if (!resolved || isOpenClawCliImageCachePath(resolved)) { + return { + aliases: [fact.path, fact.url].filter((value): value is string => Boolean(value)), + detect: false, + factIndex, + raw: identity, + type: "path", + resolved: identity, + hydrate: false, + ...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}), + }; + } + return { + aliases: [fact.path, fact.url, resolved].filter((value): value is string => Boolean(value)), + factIndex, + raw: mediaUri ?? fact.path ?? fact.url ?? resolved, + type: mediaUri ? "media-uri" : "path", + resolved, + hydrate, + ...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}), + }; +} + +export function collectMediaImageRefs( + media?: readonly MediaFact[], +): Array { + return normalizeMediaFacts(media).flatMap((fact, factIndex) => + isImageMediaFact(fact) ? [mediaFactToImageRef(fact, factIndex)] : [], + ); +} + +export function collectIdentitylessMediaImageFactIndexes(media?: readonly MediaFact[]): number[] { + return normalizeMediaFacts(media).flatMap((fact, factIndex) => + isImageMediaFact(fact) && + fact.hydrationSuppressed !== true && + fact.path === undefined && + fact.url === undefined + ? [factIndex] + : [], + ); +} + +// Guards for transports that cannot carry attachments (paired-node CLI): only +// facts that will actually hydrate an image count; described/remote-only facts +// whose hydration is suppressed must not block text-only prompts. +export function hasHydratableMediaImages(media?: readonly MediaFact[]): boolean { + return collectMediaImageRefs(media).some((ref) => ref?.hydrate === true); +} + +export function selectMediaImageRefs(params: { + refs: Array; + existingImageCount: number; + imageOrder?: readonly PromptImageOrderEntry[]; +}): Array { + const { refs } = params; + if (!params.imageOrder?.length) { + // Legacy turns (no layout metadata): identity-less facts are the inline + // images' own slots — pair them positionally so they cannot count as failed + // offloads; identity-bearing refs remain genuine offloaded attachments. + let inlinePairs = params.existingImageCount; + return refs.filter((ref) => { + if (ref === undefined && inlinePairs > 0) { + inlinePairs -= 1; + return false; + } + return true; + }); + } + if (refs.length !== params.imageOrder.length) { + // Partial fact arrays cannot prove positional ownership. Keep every ref as + // an offload so no attachment is silently consumed by an inline slot. + return refs; + } + let remainingExisting = params.existingImageCount; + return params.imageOrder.flatMap((entry, index) => { + if (entry === "offloaded") { + return [refs[index]]; + } + if (remainingExisting > 0) { + remainingExisting -= 1; + return []; + } + return [undefined]; + }); +} diff --git a/src/agents/embedded-agent-runner/run/images.replay.test.ts b/src/agents/embedded-agent-runner/run/images.replay.test.ts new file mode 100644 index 000000000000..19daff6a3adc --- /dev/null +++ b/src/agents/embedded-agent-runner/run/images.replay.test.ts @@ -0,0 +1,680 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { createSolidPngBuffer } from "../../../../test/helpers/image-fixtures.js"; +import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js"; +import { + attachRuntimePromptMediaFacts, + readRuntimePromptImageOrder, + readRuntimePromptMediaFacts, +} from "../../../media/media-facts.js"; +import { + finalizeRuntimePromptImages, + readRuntimePromptImageFactIndexes, +} from "../../../media/runtime-prompt-image-provenance.js"; +import { buildPersistedUserTurnMessage } from "../../../sessions/user-turn-transcript.js"; +import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js"; +import type { AgentMessage } from "../../runtime/index.js"; +import { + detectAndLoadPromptImages, + detectImageReferences, + hydratePromptMediaMessages, +} from "./images.js"; + +const TINY_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg=="; + +describe("structured prompt media replay", () => { + it("keeps per-slot provenance when the same image object is reused", () => { + const sharedImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const { images } = finalizeRuntimePromptImages([ + { image: sharedImage, factIndex: 0 }, + { image: sharedImage, factIndex: 1 }, + ]); + + expect(readRuntimePromptImageFactIndexes(images)).toEqual([0, 1]); + }); + + it("retains the runtime fact carrier when queued hydration fails", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-runtime-failure-")); + const media = [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }]; + const message = attachRuntimePromptMediaFacts( + { role: "user" as const, content: "missing attachment" }, + media, + ["offloaded"], + ) as unknown as AgentMessage; + const runtimeMedia = readRuntimePromptMediaFacts(message); + + try { + const result = await hydratePromptMediaMessages([message], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect(readRuntimePromptMediaFacts(result[0] as AgentMessage)).toEqual(runtimeMedia); + expect(readRuntimePromptImageOrder(result[0] as AgentMessage)).toEqual(["offloaded"]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("does not fail a described remote-only fact with no local identity", async () => { + const media = buildInboundMediaNoteProjection({ + MediaPaths: [""], + MediaUrls: ["https://example.com/described.png"], + MediaTypes: ["image/png"], + MediaUnderstanding: [ + { + kind: "image.description", + attachmentIndex: 0, + text: "already described", + provider: "test", + }, + ], + }).media; + + const result = await detectAndLoadPromptImages({ + prompt: "already described", + media, + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + }); + + expect(result.failedMediaCount).toBe(0); + expect(result.detectedRefs).toEqual([]); + expect(result.images).toEqual([]); + }); + + it("reports a fact-owned image dropped during sanitization", async () => { + const result = await detectAndLoadPromptImages({ + prompt: "inspect it", + media: [{ path: "/tmp/already-materialized.png", contentType: "image/png" }], + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }], + existingImageFactIndexes: [0], + imageOrder: ["inline"], + }); + + expect(result.failedMediaCount).toBe(1); + expect(result.images).toEqual([]); + }); + + it("preserves persisted facts when replay hydration fails", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-failure-")); + const missingPath = path.join(workspaceDir, "missing.png"); + const message = { + role: "user" as const, + content: "missing attachment", + MediaPath: missingPath, + MediaPaths: [missingPath], + MediaType: "image/png", + MediaTypes: ["image/png"], + } as unknown as AgentMessage; + + try { + const result = await hydratePromptMediaMessages([message], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + const replayed = result[0] as unknown as Record; + expect(replayed.MediaPaths).toEqual([missingPath]); + expect(replayed.content).toEqual([{ type: "text", text: "missing attachment" }]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("detects bracketed explicit paths but not textual claim-check tickets", () => { + expect(detectImageReferences("inspect [source /tmp/photo.png]")).toEqual([ + { raw: "/tmp/photo.png", type: "path", resolved: "/tmp/photo.png" }, + ]); + expect(detectImageReferences("[media attached: media://inbound/legacy.png]")).toEqual([]); + expect(detectImageReferences("[media attached: /tmp/legacy.png (image/png)]")).toEqual([]); + expect(detectImageReferences("[Image: source: /tmp/legacy.png]")).toEqual([]); + }); + + it("dedupes the rendered alias of a private macOS path", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + try { + const result = await detectAndLoadPromptImages({ + prompt: "[media attached: /var/tmp/aliased.png (image/png)]", + media: [{ path: "/private/var/tmp/aliased.png", contentType: "image/png" }], + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + }); + + expect(result.detectedRefs).toEqual([ + { + raw: "/private/var/tmp/aliased.png", + resolved: "/private/var/tmp/aliased.png", + type: "path", + }, + ]); + expect(result.skippedCount).toBe(1); + } finally { + platformSpy.mockRestore(); + } + }); + + it("keeps private-var and var paths distinct away from Darwin", () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + try { + expect( + detectImageReferences("compare /private/var/tmp/a.png and /var/tmp/a.png"), + ).toHaveLength(2); + } finally { + platformSpy.mockRestore(); + } + }); + + it("expands a home-relative path carried by a media fact", async () => { + const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-fact-home-")); + const imagePath = path.join(homeDir, "Pictures", "photo.png"); + await fs.mkdir(path.dirname(imagePath), { recursive: true }); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const envSnapshot = captureEnv(["HOME"]); + setTestEnvValue("HOME", homeDir); + try { + const result = await detectAndLoadPromptImages({ + prompt: "describe it", + media: [{ path: "~/Pictures/photo.png", contentType: "image/png" }], + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + localRoots: [homeDir], + }); + expect(result.images).toHaveLength(1); + } finally { + envSnapshot.restore(); + await fs.rm(homeDir, { recursive: true, force: true }); + } + }); + + it("keeps same-spelled relative refs from different workspaces distinct", async () => { + const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relative-roots-")); + const stagedDir = path.join(rootDir, "staged"); + const currentDir = path.join(rootDir, "current"); + await fs.mkdir(stagedDir, { recursive: true }); + await fs.mkdir(currentDir, { recursive: true }); + await fs.writeFile(path.join(stagedDir, "photo.png"), Buffer.from(TINY_PNG_BASE64, "base64")); + await fs.writeFile( + path.join(currentDir, "photo.png"), + createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 }), + ); + + try { + const result = await detectAndLoadPromptImages({ + prompt: "compare ./photo.png", + media: [{ path: "./photo.png", contentType: "image/png", workspaceDir: stagedDir }], + workspaceDir: currentDir, + model: { input: ["text", "image"] }, + localRoots: [stagedDir, currentDir], + }); + expect(result.loadedCount).toBe(2); + expect(result.images).toHaveLength(2); + } finally { + await fs.rm(rootDir, { recursive: true, force: true }); + } + }); + + it("does not duplicate an already-materialized offloaded slot", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-materialized-offload-")); + const imagePath = path.join(workspaceDir, "offloaded.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const image = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const persisted = buildPersistedUserTurnMessage({ + text: "already materialized", + media: [{ path: imagePath, contentType: "image/png" }], + mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] }, + }) as unknown as AgentMessage; + + try { + const first = await hydratePromptMediaMessages([persisted], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + const serialized = JSON.stringify(first[0]); + const restored = JSON.parse(serialized) as AgentMessage; + await fs.rm(imagePath); + const replay = await hydratePromptMediaMessages([restored], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((replay[0] as unknown as { content?: unknown }).content).toEqual([ + { type: "text", text: "already materialized" }, + image, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("prefers persisted fact-index layout when runtime order is also present", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-layout-authority-")); + const describedPath = path.join(workspaceDir, "described.png"); + const inlinePath = path.join(workspaceDir, "inline.png"); + const offloadedPath = path.join(workspaceDir, "offloaded.png"); + const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 }); + await fs.writeFile(describedPath, createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 })); + await fs.writeFile(inlinePath, Buffer.from(TINY_PNG_BASE64, "base64")); + await fs.writeFile(offloadedPath, offloadedBuffer); + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const media = [ + { path: describedPath, contentType: "image/png" }, + { path: inlinePath, contentType: "image/png" }, + { path: offloadedPath, contentType: "image/png" }, + ]; + const persisted = buildPersistedUserTurnMessage({ + text: "compare", + media, + mediaImageLayout: { + slots: [ + { kind: "inline", factIndex: 1 }, + { kind: "offloaded", factIndex: 2 }, + ], + suppressedFactIndexes: [0], + }, + }); + const message = attachRuntimePromptMediaFacts( + { + ...persisted, + content: [{ type: "text" as const, text: "compare" }, inlineImage], + } as AgentMessage, + media, + ["inline", "offloaded"], + ); + + try { + const first = await hydratePromptMediaMessages([message], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + const firstContent = (first[0] as unknown as { content?: unknown[] }).content; + expect(firstContent).toEqual([ + { type: "text", text: "compare" }, + inlineImage, + { type: "image", data: offloadedBuffer.toString("base64"), mimeType: "image/png" }, + ]); + + const serialized = JSON.stringify(first[0]); + await fs.rm(describedPath); + await fs.rm(inlinePath); + await fs.rm(offloadedPath); + const replay = await hydratePromptMediaMessages([JSON.parse(serialized) as AgentMessage], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((replay[0] as unknown as { content?: unknown[] }).content).toEqual(firstContent); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("retries an offloaded fact even when an unrelated explicit image exists", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-offload-retry-")); + const explicitImage = { + type: "image" as const, + data: TINY_PNG_BASE64, + mimeType: "image/png", + }; + try { + const result = await detectAndLoadPromptImages({ + prompt: "retry missing attachment", + media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }], + mediaImageLayout: { + slots: [{ kind: "offloaded", factIndex: 0 }], + suppressedFactIndexes: [], + }, + workspaceDir, + model: { input: ["text", "image"] }, + existingImages: [explicitImage], + existingImageFactIndexes: [null], + workspaceOnly: true, + }); + expect(result.failedMediaCount).toBe(1); + expect(result.images).toEqual([explicitImage]); + expect(result.imageFactIndexes).toEqual([null]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("does not assign a partial offloaded fact to an existing inline image", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-facts-")); + const offloadedPath = path.join(workspaceDir, "offloaded.png"); + const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 }); + await fs.writeFile(offloadedPath, offloadedBuffer); + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + + try { + const result = await detectAndLoadPromptImages({ + prompt: "compare", + media: [{ path: offloadedPath, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + existingImages: [inlineImage], + imageOrder: ["inline", "offloaded"], + workspaceOnly: true, + }); + expect(result.failedMediaCount).toBe(0); + expect(result.loadedCount).toBe(1); + expect(result.images).toEqual([ + inlineImage, + { type: "image", data: offloadedBuffer.toString("base64"), mimeType: "image/png" }, + ]); + expect(result.imageFactIndexes).toEqual([null, 0]); + expect(readRuntimePromptImageFactIndexes(result.images)).toEqual([null, 0]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("preserves fact order for partial hydration without persisted layout", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-order-")); + const firstPath = path.join(workspaceDir, "first.png"); + const secondPath = path.join(workspaceDir, "second.png"); + const thirdPath = path.join(workspaceDir, "third.png"); + const secondBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 }); + const thirdBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 }); + await fs.writeFile(thirdPath, thirdBuffer); + const firstImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const secondImage = { + type: "image" as const, + data: secondBuffer.toString("base64"), + mimeType: "image/png", + }; + + try { + const result = await detectAndLoadPromptImages({ + prompt: "compare", + media: [ + { path: firstPath, contentType: "image/png" }, + { path: secondPath, contentType: "image/png" }, + { path: thirdPath, contentType: "image/png" }, + ], + workspaceDir, + model: { input: ["text", "image"] }, + existingImages: [firstImage, secondImage], + existingImageFactIndexes: [0, 1], + imageOrder: ["offloaded", "inline", "offloaded"], + workspaceOnly: true, + }); + + expect(result.failedMediaCount).toBe(0); + expect(result.images).toEqual([ + firstImage, + secondImage, + { type: "image", data: thirdBuffer.toString("base64"), mimeType: "image/png" }, + ]); + expect(result.imageFactIndexes).toEqual([0, 1, 2]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("uses an unowned block for an inline slot when exact provenance is unavailable", async () => { + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const offloadedImage = { + type: "image" as const, + data: createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 }).toString("base64"), + mimeType: "image/png", + }; + + const result = await detectAndLoadPromptImages({ + prompt: "compare", + media: [{ kind: "document" }, { kind: "image" }, { kind: "image" }], + mediaImageLayout: { + slots: [ + { kind: "inline", factIndex: 1 }, + { kind: "offloaded", factIndex: 2 }, + ], + suppressedFactIndexes: [], + }, + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + existingImages: [inlineImage, offloadedImage], + existingImageFactIndexes: [null, 2], + }); + + expect(result.failedMediaCount).toBe(0); + expect(result.images).toEqual([inlineImage, offloadedImage]); + expect(result.imageFactIndexes).toEqual([null, 2]); + }); + + it("uses explicit inline layout ownership for sanitization failures", async () => { + const result = await detectAndLoadPromptImages({ + prompt: "inspect", + media: [{ kind: "image" }, { kind: "image" }], + mediaImageLayout: { + slots: [{ kind: "inline", factIndex: 1 }], + suppressedFactIndexes: [0], + }, + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }], + }); + + expect(result.failedMediaCount).toBe(1); + expect(result.images).toEqual([]); + }); + + it("reports a missing unsuppressed inline layout slot", async () => { + const result = await detectAndLoadPromptImages({ + prompt: "inspect", + media: [{ kind: "image" }], + mediaImageLayout: { + slots: [{ kind: "inline", factIndex: 0 }], + suppressedFactIndexes: [], + }, + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + }); + + expect(result.failedMediaCount).toBe(1); + expect(result.images).toEqual([]); + }); + + it("pairs an identity-less legacy fact with its existing inline block", async () => { + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const message = { + role: "user" as const, + content: [{ type: "text" as const, text: "legacy identity-less" }, inlineImage], + media: [{ kind: "image" }], + } as unknown as AgentMessage; + + const result = await hydratePromptMediaMessages([message], { + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + }); + + expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([ + { type: "text", text: "legacy identity-less" }, + inlineImage, + ]); + }); + + it("does not pair an unresolved remote legacy fact with an existing inline block", async () => { + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const message = { + role: "user" as const, + content: [{ type: "text" as const, text: "remote identity" }, inlineImage], + media: [{ kind: "image", url: "https://example.test/remote.png" }], + } as unknown as AgentMessage; + + const result = await hydratePromptMediaMessages([message], { + workspaceDir: "/tmp", + model: { input: ["text", "image"] }, + }); + const meta = (result[0] as unknown as Record)["__openclaw"] as + | Record + | undefined; + + expect(meta?.mediaImageBlockFactIndexes).toEqual([null]); + expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([ + { type: "text", text: "remote identity" }, + inlineImage, + ]); + }); + + it("hydrates an identity-bearing legacy fact beside an unowned inline block", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-legacy-inline-")); + const imagePath = path.join(workspaceDir, "inline.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const message = { + role: "user" as const, + content: [{ type: "text" as const, text: "legacy" }, inlineImage], + MediaPath: imagePath, + MediaPaths: [imagePath], + MediaType: "image/png", + MediaTypes: ["image/png"], + } as unknown as AgentMessage; + + try { + const result = await hydratePromptMediaMessages([message], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((result[0] as unknown as { content?: unknown }).content).toEqual([ + { type: "text", text: "legacy" }, + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + inlineImage, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("keeps identity-bearing legacy facts ahead of an unowned inline block", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-legacy-order-")); + const existingPath = path.join(workspaceDir, "existing.png"); + const hydratedPath = path.join(workspaceDir, "hydrated.png"); + const hydratedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 }); + await fs.writeFile(existingPath, Buffer.from(TINY_PNG_BASE64, "base64")); + await fs.writeFile(hydratedPath, hydratedBuffer); + const existingImage = { + type: "image" as const, + data: TINY_PNG_BASE64, + mimeType: "image/png", + }; + const message = { + role: "user" as const, + content: [{ type: "text" as const, text: "legacy order" }, existingImage], + MediaPath: existingPath, + MediaPaths: [existingPath, hydratedPath], + MediaType: "image/png", + MediaTypes: ["image/png", "image/png"], + } as unknown as AgentMessage; + + try { + const result = await hydratePromptMediaMessages([message], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([ + { type: "text", text: "legacy order" }, + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + { type: "image", data: hydratedBuffer.toString("base64"), mimeType: "image/png" }, + existingImage, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("dedupes the local path alias of a claim-check fact", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-alias-")); + const workspaceDir = path.join(stateDir, "workspace"); + const inboundDir = path.join(stateDir, "media", "inbound"); + const mediaId = "aliased.png"; + const imagePath = path.join(inboundDir, mediaId); + await fs.mkdir(workspaceDir, { recursive: true }); + await fs.mkdir(inboundDir, { recursive: true }); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + + try { + const result = await detectAndLoadPromptImages({ + prompt: `[media attached: ${imagePath} (image/png)]`, + media: [{ path: imagePath, url: `media://inbound/${mediaId}`, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + }); + + expect(result.loadedCount).toBe(1); + expect(result.images).toHaveLength(1); + expect(result.detectedRefs).toEqual([ + { + raw: `media://inbound/${mediaId}`, + resolved: `media://inbound/${mediaId}`, + type: "media-uri", + }, + ]); + } finally { + envSnapshot.restore(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + + it("keeps described image facts suppressed across serialize and restore", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-suppressed-")); + const imagePath = path.join(workspaceDir, "described.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const serialized = JSON.stringify( + buildPersistedUserTurnMessage({ + text: "description already present", + media: [ + { path: imagePath, contentType: "image/png", hydrationSuppressed: true }, + { kind: "sticker", hydrationSuppressed: true }, + ], + }), + ); + const restored = JSON.parse(serialized) as AgentMessage; + const meta = (restored as unknown as Record)["__openclaw"] as + | Record + | undefined; + const persistedMedia = meta?.media as + | Array<{ path?: string; contentType?: string; hydrationSuppressed?: boolean }> + | undefined; + + try { + expect(persistedMedia).toEqual([ + expect.objectContaining({ + path: imagePath, + contentType: "image/png", + hydrationSuppressed: true, + }), + expect.objectContaining({ kind: "sticker", hydrationSuppressed: true }), + ]); + const replayHydration = await detectAndLoadPromptImages({ + prompt: "description already present", + media: persistedMedia, + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect(replayHydration.failedMediaCount).toBe(0); + expect(replayHydration.images).toEqual([]); + + const result = await hydratePromptMediaMessages([restored], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((result[0] as unknown as { content?: unknown }).content).toEqual([ + { type: "text", text: "description already present" }, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/agents/embedded-agent-runner/run/images.test.ts b/src/agents/embedded-agent-runner/run/images.test.ts index cd26b906ceb9..2c7067ba5914 100644 --- a/src/agents/embedded-agent-runner/run/images.test.ts +++ b/src/agents/embedded-agent-runner/run/images.test.ts @@ -5,11 +5,24 @@ import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { describe, expect, it, vi } from "vitest"; +import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js"; import { resolvePreferredOpenClawTmpDir } from "../../../infra/tmp-openclaw-dir.js"; +import { + attachRuntimePromptMediaFacts, + readRuntimePromptImageOrder, +} from "../../../media/media-facts.js"; +import { + buildPersistedUserTurnMessage, + mergePreparedUserTurnMessageForRuntime, +} from "../../../sessions/user-turn-transcript.js"; import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js"; -import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js"; +import type { AgentMessage } from "../../runtime/index.js"; import { createUnsafeMountedSandbox } from "../../test-helpers/unsafe-mounted-sandbox.js"; -import { detectAndLoadPromptImages, detectImageReferences, loadImageFromRef } from "./images.js"; +import { + detectAndLoadPromptImages, + detectImageReferences, + hydratePromptMediaMessages, +} from "./images.js"; const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg=="; @@ -154,10 +167,10 @@ describe("detectImageReferences", () => { }); it("does not leak parser state between calls", () => { - expect(detectImageReferences("[media attached: /tmp/first.png (image/png)]")).toStrictEqual([ + expect(detectImageReferences("See /tmp/first.png")).toStrictEqual([ { raw: "/tmp/first.png", type: "path", resolved: "/tmp/first.png" }, ]); - expect(detectImageReferences("[Image: source: /tmp/second.jpg]")).toStrictEqual([ + expect(detectImageReferences("See /tmp/second.jpg")).toStrictEqual([ { raw: "/tmp/second.jpg", type: "path", resolved: "/tmp/second.jpg" }, ]); const thirdPath = path.join(os.tmpdir(), "third.webp"); @@ -265,67 +278,6 @@ describe("detectImageReferences", () => { }); }); - it("detects [Image: source: ...] format from messaging systems", () => { - const ref = expectSingleImageReference(`What does this image show? -[Image: source: /Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg]`); - - expect(ref).toStrictEqual({ - raw: "/Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg", - type: "path", - resolved: "/Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg", - }); - }); - - it("handles complex message attachment paths", () => { - const ref = expectSingleImageReference( - "[Image: source: /Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg]", - ); - - expect(ref).toStrictEqual({ - raw: "/Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg", - type: "path", - resolved: - "/Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg", - }); - }); - - it("detects multiple images in [media attached: ...] format", () => { - // Multi-file format uses separate brackets on separate lines - const refs = expectImageReferenceCount( - `[media attached: 2 files] -[media attached 1/2: /Users/tyleryust/.openclaw/media/IMG_6430.jpeg (image/jpeg)] -[media attached 2/2: /Users/tyleryust/.openclaw/media/IMG_6431.jpeg (image/jpeg)] -what about these images?`, - 2, - ); - - expect(refs).toStrictEqual([ - { - raw: "/Users/tyleryust/.openclaw/media/IMG_6430.jpeg", - type: "path", - resolved: "/Users/tyleryust/.openclaw/media/IMG_6430.jpeg", - }, - { - raw: "/Users/tyleryust/.openclaw/media/IMG_6431.jpeg", - type: "path", - resolved: "/Users/tyleryust/.openclaw/media/IMG_6431.jpeg", - }, - ]); - }); - - it("does not double-count path and url in same bracket", () => { - // Single file with URL (| separates path from url, not multiple files) - const ref = expectSingleImageReference( - "[media attached: /cache/IMG_6430.jpeg (image/jpeg) | /cache/IMG_6430.jpeg]", - ); - - expect(ref).toStrictEqual({ - raw: "/cache/IMG_6430.jpeg", - type: "path", - resolved: "/cache/IMG_6430.jpeg", - }); - }); - it("ignores remote URLs entirely (local-only)", () => { const refs = expectImageReferenceCount( `To send an image: MEDIA:https://example.com/image.jpg @@ -343,31 +295,6 @@ Also https://cdn.mysite.com/img.jpg`, ]); }); - it("handles single file format with URL (no index)", () => { - const ref = - expectSingleImageReference(`[media attached: /cache/photo.jpeg (image/jpeg) | https://example.com/url] -what is this?`); - - expect(ref).toStrictEqual({ - raw: "/cache/photo.jpeg", - type: "path", - resolved: "/cache/photo.jpeg", - }); - }); - - it("handles paths with spaces in filename", () => { - // URL after | is https, not a local path, so only the local path should be detected - const ref = - expectSingleImageReference(`[media attached: /Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png (image/png) | https://example.com/same.png] -what is this?`); - - expect(ref).toStrictEqual({ - raw: "/Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png", - type: "path", - resolved: "/Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png", - }); - }); - it("ignores remote-host file URLs", () => { expectNoImageReferences("See file://attacker/share/evil.png"); }); @@ -385,107 +312,6 @@ what is this?`); }); }); -describe("loadImageFromRef", () => { - it("hydrates managed inbound media URIs before workspace path resolution", async () => { - // Managed media URIs are canonical inbound attachment handles and should - // work even when workspaceOnly would reject ordinary outside paths. - const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-uri-")); - const workspaceDir = path.join(stateDir, "workspace-agent"); - const inboundDir = path.join(stateDir, "media", "inbound"); - const mediaId = "telegram-photo.png"; - await fs.mkdir(workspaceDir, { recursive: true }); - await fs.mkdir(inboundDir, { recursive: true }); - await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64")); - const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); - setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); - - try { - const image = await loadImageFromRef( - { - raw: `media://inbound/${mediaId}`, - type: "media-uri", - resolved: `media://inbound/${mediaId}`, - }, - workspaceDir, - { workspaceOnly: true }, - ); - - expect(image?.type).toBe("image"); - expect(image?.mimeType).toBe("image/png"); - expect(image?.data).toBe(TINY_PNG_BASE64); - } finally { - envSnapshot.restore(); - await fs.rm(stateDir, { recursive: true, force: true }); - } - }); - - it("hydrates sandbox-staged inbound media URIs", async () => { - const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-sbx-uri-")); - const inboundDir = path.join(sandboxRoot, "media", "inbound"); - const mediaId = "telegram-photo.png"; - await fs.mkdir(inboundDir, { recursive: true }); - await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64")); - - try { - const image = await loadImageFromRef( - { - raw: `media://inbound/${mediaId}`, - type: "media-uri", - resolved: `media://inbound/${mediaId}`, - }, - sandboxRoot, - { - workspaceOnly: true, - sandbox: { - root: sandboxRoot, - bridge: createHostSandboxFsBridge(sandboxRoot), - }, - }, - ); - - expect(image?.type).toBe("image"); - expect(image?.mimeType).toBe("image/png"); - expect(image?.data).toBe(TINY_PNG_BASE64); - } finally { - await fs.rm(sandboxRoot, { recursive: true, force: true }); - } - }); - - it("allows sandbox-validated host paths outside default media roots", async () => { - const homeDir = os.homedir(); - await fs.mkdir(homeDir, { recursive: true }); - const sandboxParent = await fs.mkdtemp(path.join(homeDir, "openclaw-sandbox-image-")); - try { - const sandboxRoot = path.join(sandboxParent, "sandbox"); - await fs.mkdir(sandboxRoot, { recursive: true }); - const imagePath = path.join(sandboxRoot, "photo.png"); - const pngB64 = TINY_PNG_BASE64; - await fs.writeFile(imagePath, Buffer.from(pngB64, "base64")); - - const image = await loadImageFromRef( - { - raw: "./photo.png", - type: "path", - resolved: "./photo.png", - }, - sandboxRoot, - { - sandbox: { - root: sandboxRoot, - bridge: createHostSandboxFsBridge(sandboxRoot), - }, - }, - ); - - expect(image?.type).toBe("image"); - expect(image?.mimeType).toBe("image/png"); - expect(image?.data).toBe(TINY_PNG_BASE64); - } finally { - await fs.rm(sandboxParent, { recursive: true, force: true }); - } - }); -}); - describe("detectAndLoadPromptImages", () => { it("returns no images for non-vision models even when existing images are provided", async () => { const result = await detectAndLoadPromptImages({ @@ -529,6 +355,7 @@ describe("detectAndLoadPromptImages", () => { try { const result = await detectAndLoadPromptImages({ prompt: "[media attached: ./photo.png (image/png)]\ndescribe it", + media: [{ path: imagePath, contentType: "image/png" }], workspaceDir: stateDir, model: { input: ["text", "image"] }, existingImages: [{ type: "image", data: pngB64, mimeType: "image/png" }], @@ -536,7 +363,7 @@ describe("detectAndLoadPromptImages", () => { workspaceOnly: true, }); - expect(result.detectedRefs).toHaveLength(1); + expect(result.detectedRefs).toEqual([{ raw: imagePath, type: "path", resolved: imagePath }]); expect(result.loadedCount).toBe(0); expect(result.skippedCount).toBe(0); expect(result.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]); @@ -545,6 +372,65 @@ describe("detectAndLoadPromptImages", () => { } }); + it("uses a described fact identity to suppress its generated media-note path", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-described-dedupe-")); + const imagePath = path.join(workspaceDir, "photo.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + + try { + const result = await detectAndLoadPromptImages({ + prompt: `[media attached: ${imagePath} (image/png)]`, + media: buildInboundMediaNoteProjection({ + MediaPath: imagePath, + MediaType: "image/png", + MediaUnderstanding: [ + { + kind: "image.description", + attachmentIndex: 0, + text: "already described", + provider: "test", + }, + ], + }).media, + workspaceDir, + model: { input: ["text", "image"] }, + }); + + expect(result.detectedRefs).toEqual([{ raw: imagePath, type: "path", resolved: imagePath }]); + expect(result.loadedCount).toBe(0); + expect(result.images).toEqual([]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("dedupes a relative fact projection against the fact workspace", async () => { + const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-fact-workspace-dedupe-")); + const stagedDir = path.join(rootDir, "staged"); + const currentDir = path.join(rootDir, "current"); + await fs.mkdir(stagedDir, { recursive: true }); + await fs.mkdir(currentDir, { recursive: true }); + await fs.writeFile(path.join(stagedDir, "photo.png"), Buffer.from(TINY_PNG_BASE64, "base64")); + await fs.writeFile(path.join(currentDir, "photo.png"), TINY_GIF_BUFFER); + + try { + const result = await detectAndLoadPromptImages({ + prompt: "[media attached: ./photo.png (image/png)]", + media: [{ path: "./photo.png", contentType: "image/png", workspaceDir: stagedDir }], + workspaceDir: currentDir, + model: { input: ["text", "image"] }, + localRoots: [stagedDir, currentDir], + }); + + expect(result.loadedCount).toBe(1); + expect(result.images).toEqual([ + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + ]); + } finally { + await fs.rm(rootDir, { recursive: true, force: true }); + } + }); + it("keeps distinct inline attachments with identical bytes", async () => { const pngB64 = TINY_PNG_BASE64; const image = { type: "image" as const, data: pngB64, mimeType: "image/png" }; @@ -561,6 +447,37 @@ describe("detectAndLoadPromptImages", () => { expect(result.images).toEqual([image, image]); }); + it("keeps offloaded-only facts when existing images have no order metadata", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-unordered-images-")); + const imagePath = path.join(workspaceDir, "offloaded.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const inlineImage = { + type: "image" as const, + data: TINY_GIF_BUFFER.toString("base64"), + mimeType: "image/gif", + }; + + try { + const result = await detectAndLoadPromptImages({ + prompt: "compare", + media: [{ path: imagePath, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + existingImages: [inlineImage], + imageOrder: [], + workspaceOnly: true, + }); + + expect(result.loadedCount).toBe(1); + expect(result.images).toEqual([ + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + inlineImage, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + it("classifies prompt and attachment refs while preserving mixed attachment order", async () => { const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-order-")); const inboundDir = path.join(stateDir, "media", "inbound"); @@ -579,6 +496,7 @@ describe("detectAndLoadPromptImages", () => { try { const result = await detectAndLoadPromptImages({ prompt, + media: [{ url: "media://inbound/att-b.gif", contentType: "image/gif" }], workspaceDir: stateDir, model: { input: ["text", "image"] }, existingImages: [{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }], @@ -587,11 +505,6 @@ describe("detectAndLoadPromptImages", () => { }); expect(result.detectedRefs).toEqual([ - { - raw: "media://inbound/prompt-ref.png", - type: "media-uri", - resolved: "media://inbound/prompt-ref.png", - }, { raw: "media://inbound/att-b.gif", type: "media-uri", @@ -599,12 +512,11 @@ describe("detectAndLoadPromptImages", () => { }, { raw: "./prompt-b.png", type: "path", resolved: "./prompt-b.png" }, ]); - expect(result.loadedCount).toBe(3); + expect(result.loadedCount).toBe(2); expect(result.images).toEqual([ { type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" }, { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, - { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, ]); } finally { envSnapshot.restore(); @@ -612,6 +524,63 @@ describe("detectAndLoadPromptImages", () => { } }); + it("preserves an empty described-image slot in mixed attachment order", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-described-image-order-")); + const inboundDir = path.join(stateDir, "media", "inbound"); + const mediaId = "remaining.gif"; + await fs.mkdir(inboundDir, { recursive: true }); + await fs.writeFile(path.join(inboundDir, mediaId), TINY_GIF_BUFFER); + const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + + try { + const result = await detectAndLoadPromptImages({ + prompt: "compare", + media: [ + { kind: "image" }, + { kind: "image" }, + { url: `media://inbound/${mediaId}`, contentType: "image/gif" }, + ], + workspaceDir: stateDir, + model: { input: ["text", "image"] }, + existingImages: [inlineImage], + imageOrder: ["offloaded", "inline", "offloaded"], + }); + + expect(result.images).toEqual([ + inlineImage, + { type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" }, + ]); + } finally { + envSnapshot.restore(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + + it("does not load an explicit prompt ref twice when the same fact owns it", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-fact-")); + await fs.writeFile(path.join(workspaceDir, "same.png"), Buffer.from(TINY_PNG_BASE64, "base64")); + + try { + const result = await detectAndLoadPromptImages({ + prompt: "Compare ./same.png", + media: [{ path: "./same.png", contentType: "image/png", workspaceDir }], + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + + expect(result.detectedRefs).toEqual([ + { raw: "./same.png", type: "path", resolved: "./same.png" }, + ]); + expect(result.loadedCount).toBe(1); + expect(result.images).toHaveLength(1); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + it("blocks prompt image refs outside workspace when sandbox workspaceOnly is enabled", async () => { // Sandbox workspaceOnly uses the bridge to validate mounted paths; ordinary // prompt refs outside the workspace are detected but intentionally skipped. @@ -675,4 +644,220 @@ describe("detectAndLoadPromptImages", () => { await fs.rm(stateDir, { recursive: true, force: true }); } }); + + it("keeps the fact hydration size limit", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-size-")); + const imagePath = path.join(workspaceDir, "too-large.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + + try { + const result = await detectAndLoadPromptImages({ + prompt: "describe it", + media: [{ path: imagePath, contentType: "image/png" }], + workspaceDir, + model: { input: ["text", "image"] }, + maxBytes: 1, + }); + + expect(result.loadedCount).toBe(0); + expect(result.failedMediaCount).toBe(1); + expect(result.skippedCount).toBe(1); + expect(result.images).toHaveLength(0); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); +}); + +describe("hydratePromptMediaMessages", () => { + it("hydrates queued facts in attachment order without mutating cache-stable input", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-queued-image-facts-")); + const inboundDir = path.join(stateDir, "media", "inbound"); + const mediaId = "queued.gif"; + await fs.mkdir(inboundDir, { recursive: true }); + await fs.writeFile(path.join(inboundDir, mediaId), TINY_GIF_BUFFER); + const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const message = attachRuntimePromptMediaFacts( + { role: "user" as const, content: [{ type: "text" as const, text: "compare" }, inlineImage] }, + [{ url: `media://inbound/${mediaId}`, contentType: "image/gif" }], + ["offloaded", "inline"], + ); + const options = { + workspaceDir: stateDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }; + + try { + const runtimeMessage = message as unknown as AgentMessage; + const first = await hydratePromptMediaMessages([runtimeMessage], options); + const second = await hydratePromptMediaMessages([runtimeMessage], options); + const firstContent = (first?.[0] as unknown as { content?: unknown })?.content; + const secondContent = (second?.[0] as unknown as { content?: unknown })?.content; + + expect(firstContent).toEqual([ + { type: "text", text: "compare" }, + { type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" }, + inlineImage, + ]); + expect(secondContent).toEqual(firstContent); + expect(message.content).toEqual([{ type: "text", text: "compare" }, inlineImage]); + } finally { + envSnapshot.restore(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + + it("hydrates a fact-owned user message whose content is a string", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-string-image-facts-")); + const imagePath = path.join(workspaceDir, "photo.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const message = attachRuntimePromptMediaFacts( + { role: "user" as const, content: "describe it" }, + [{ path: imagePath, contentType: "image/png" }], + ) as unknown as AgentMessage; + + try { + const result = await hydratePromptMediaMessages([message], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((result[0] as unknown as { content?: unknown }).content).toEqual([ + { type: "text", text: "describe it" }, + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("reconstructs recent facts from serialized transcript media fields", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replayed-image-")); + const imagePath = path.join(workspaceDir, "photo.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const message = { + role: "user" as const, + content: "describe the replayed image", + MediaPath: imagePath, + MediaPaths: [imagePath], + MediaType: "image/png", + MediaTypes: ["image/png"], + } as unknown as AgentMessage; + + try { + const result = await hydratePromptMediaMessages([message], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((result[0] as unknown as { content?: unknown }).content).toEqual([ + { type: "text", text: "describe the replayed image" }, + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("preserves offloaded-before-inline order across serialize and restore", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-order-")); + const offloadedPath = path.join(workspaceDir, "offloaded.png"); + const inlinePath = path.join(workspaceDir, "inline.gif"); + await fs.writeFile(offloadedPath, Buffer.from(TINY_PNG_BASE64, "base64")); + await fs.writeFile(inlinePath, TINY_GIF_BUFFER); + const inlineImage = { + type: "image" as const, + data: TINY_GIF_BUFFER.toString("base64"), + mimeType: "image/gif", + }; + const runtime = attachRuntimePromptMediaFacts( + { role: "user" as const, content: [{ type: "text" as const, text: "compare" }, inlineImage] }, + [{ path: offloadedPath, contentType: "image/png" }], + ["offloaded", "inline"], + ) as unknown as AgentMessage; + const persisted = buildPersistedUserTurnMessage({ + text: "compare", + media: [ + { path: offloadedPath, contentType: "image/png" }, + { path: inlinePath, contentType: "image/gif" }, + ], + mediaImageLayout: { + slots: [ + { kind: "offloaded", factIndex: 0 }, + { kind: "inline", factIndex: 1 }, + ], + }, + }); + const serialized = JSON.stringify( + mergePreparedUserTurnMessageForRuntime({ + runtimeMessage: runtime, + preparedMessage: persisted, + }), + ); + const restored = JSON.parse(serialized) as AgentMessage; + + try { + expect(readRuntimePromptImageOrder(restored)).toBeUndefined(); + const result = await hydratePromptMediaMessages([restored], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((result[0] as unknown as { content?: unknown }).content).toEqual([ + { type: "text", text: "compare" }, + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + inlineImage, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("keeps duplicate fact slots across serialize and restore", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-duplicates-")); + const imagePath = path.join(workspaceDir, "same.png"); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const persisted = buildPersistedUserTurnMessage({ + text: "compare duplicates", + media: [ + { path: imagePath, contentType: "image/png" }, + { path: imagePath, contentType: "image/png" }, + ], + mediaImageLayout: { + slots: [ + { kind: "offloaded", factIndex: 0 }, + { kind: "inline", factIndex: 1 }, + ], + }, + }); + const serialized = JSON.stringify( + mergePreparedUserTurnMessageForRuntime({ + runtimeMessage: { + role: "user", + content: [{ type: "text", text: "compare duplicates" }, inlineImage], + } as AgentMessage, + preparedMessage: persisted, + }), + ); + const restored = JSON.parse(serialized) as AgentMessage; + + try { + const result = await hydratePromptMediaMessages([restored], { + workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + const content = (result[0] as unknown as { content?: unknown[] }).content ?? []; + expect(content.filter((block) => (block as { type?: unknown }).type === "image")).toEqual([ + inlineImage, + inlineImage, + ]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/agents/embedded-agent-runner/run/images.ts b/src/agents/embedded-agent-runner/run/images.ts index d9bad718debc..d6dac26ea899 100644 --- a/src/agents/embedded-agent-runner/run/images.ts +++ b/src/agents/embedded-agent-runner/run/images.ts @@ -1,16 +1,24 @@ -/** - * Detects, resolves, and loads prompt image references for model input. - */ import path from "node:path"; import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { formatErrorMessage } from "../../../infra/errors.js"; import { assertNoWindowsNetworkPath, safeFileURLToPath } from "../../../infra/local-file-access.js"; import type { ImageContent } from "../../../llm/types.js"; +import { + attachRuntimePromptMediaFacts, + isImageMediaFact, + normalizeMediaFacts, + readRuntimePromptImageOrder, + readRuntimePromptMediaFacts, + resolveMediaFacts, + type MediaFact, +} from "../../../media/media-facts.js"; import { resolveMediaReferenceLocalPath } from "../../../media/media-reference.js"; import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js"; +import { finalizeRuntimePromptImages } from "../../../media/runtime-prompt-image-provenance.js"; import { loadWebMedia } from "../../../media/web-media.js"; import { resolveUserPath } from "../../../utils.js"; import type { ImageSanitizationLimits } from "../../image-sanitization.js"; +import type { AgentMessage } from "../../runtime/index.js"; import { createSandboxBridgeReadFile, resolveSandboxedBridgeMediaPath, @@ -18,10 +26,25 @@ import { import type { SandboxFsBridge } from "../../sandbox/fs-bridge.js"; import { sanitizeImageBlocks } from "../../tool-images.js"; import { log } from "../logger.js"; +import { + collectIdentitylessMediaImageFactIndexes, + collectMediaImageRefs, + isOpenClawCliImageCachePath, + selectMediaImageRefs, + type MediaImageRef, +} from "./images.media-refs.js"; +import { + type ImageFactIndex, + type MediaImageLayout, + countMissingLayoutInlineSlots, + readPersistedImageBlockFactIndexes, + readPersistedMediaImageLayout, + readPersistedPromptMediaFacts, + resolveLayoutInlineFactIndexes, +} from "./prompt-image-metadata.js"; + +export { hasHydratableMediaImages } from "./images.media-refs.js"; -/** - * Common image file extensions for detection. - */ const IMAGE_EXTENSION_NAMES = [ "png", "jpg", @@ -39,307 +62,145 @@ for (const ext of IMAGE_EXTENSION_NAMES) { IMAGE_EXTENSIONS.add(`.${ext}`); } const IMAGE_EXTENSION_PATTERN = IMAGE_EXTENSION_NAMES.join("|"); -const MEDIA_ATTACHED_PATH_REGEX_SOURCE = - "^\\s*(.+?\\.(?:" + IMAGE_EXTENSION_PATTERN + "))\\s*(?:\\(|$|\\|)"; -const MESSAGE_IMAGE_REGEX_SOURCE = - "\\[Image:\\s*source:\\s*([^\\]]+\\.(?:" + IMAGE_EXTENSION_PATTERN + "))\\]"; const FILE_URL_REGEX_SOURCE = "file://[^\\s<>\"'`\\]]+\\.(?:" + IMAGE_EXTENSION_PATTERN + ")"; const WINDOWS_DRIVE_PATH_REGEX_SOURCE = "(?:^|\\s|[\"'`(])([A-Za-z]:[\\\\/][^\\s\"'`()\\[\\]]*\\.(?:" + IMAGE_EXTENSION_PATTERN + "))"; const PATH_REGEX_SOURCE = "(?:^|\\s|[\"'`(])((\\.\\.?/|[~/])[^\\s\"'`()\\[\\]]*\\.(?:" + IMAGE_EXTENSION_PATTERN + "))"; -const MEDIA_ATTACHED_PATTERN = /\[media attached(?:\s+\d+\/\d+)?:\s*([^\]]+)\]/gi; -const MEDIA_ATTACHED_PATH_PATTERN = new RegExp(MEDIA_ATTACHED_PATH_REGEX_SOURCE, "i"); -const MESSAGE_IMAGE_PATTERN = new RegExp(MESSAGE_IMAGE_REGEX_SOURCE, "gi"); const FILE_URL_PATTERN = new RegExp(FILE_URL_REGEX_SOURCE, "gi"); const WINDOWS_DRIVE_PATH_PATTERN = new RegExp(WINDOWS_DRIVE_PATH_REGEX_SOURCE, "gi"); const PATH_PATTERN = new RegExp(PATH_REGEX_SOURCE, "gi"); +const LEGACY_ATTACHMENT_MARKER_PATTERN = + /\[(?:media attached(?:\s+\d+\/\d+)?:|Image:\s*source:)\s*[^\]]+\]/gi; -/** - * Matches the opaque media URI written by the Gateway's claim-check offload: - * media://inbound/ - * - * Uses an exclusion-based character class rather than a whitelist so that - * Unicode filenames (e.g. Chinese characters) preserved by sanitizeFilename - * in store.ts are matched correctly. - * - * Explicitly excluded from the ID segment: - * ] — closes the surrounding [media attached: ...] bracket - * \s — any whitespace (space, newline, tab) — terminates the token - * / — forward slash path separator (traversal prevention) - * \ — back slash path separator (traversal prevention) - * \x00 — null byte (path injection prevention) - * - * resolveMediaBufferPath applies its own guards against these characters, but - * excluding them here provides defence-in-depth at the parsing layer. - * - * Example valid IDs: - * "1c77ce17-20b9-4546-be64-6e36a9adcb2c.png" - * "photo---1c77ce17-20b9-4546-be64-6e36a9adcb2c.png" - * "图片---1c77ce17-20b9-4546-be64-6e36a9adcb2c.png" - */ -const MEDIA_URI_REGEX = /\bmedia:\/\/inbound\/([^\]\s/\\]+)/; - -/** - * Result of detecting an image reference in text. - */ interface DetectedImageRef { - /** The raw matched string from the prompt */ raw: string; - /** The type of reference */ type: "path" | "media-uri"; - /** The resolved/normalized path, or the raw media URI for media-uri type */ resolved: string; } -/** - * Checks if a file extension indicates an image file. - */ function isImageExtension(filePath: string): boolean { const ext = normalizeLowercaseStringOrEmpty(path.extname(filePath)); return IMAGE_EXTENSIONS.has(ext); } function normalizeRefForDedupe(raw: string): string { - return process.platform === "win32" ? normalizeLowercaseStringOrEmpty(raw) : raw; + const projected = + process.platform === "darwin" && raw.startsWith("/private/var/") + ? raw.slice("/private".length) + : raw; + return process.platform === "win32" ? normalizeLowercaseStringOrEmpty(projected) : projected; } -function isOpenClawCliImageCachePath(filePath: string): boolean { - const parts = filePath.replaceAll("\\", "/").split("/"); - return parts.some((part, index) => { - if (part === ".openclaw-cli-images") { - return true; - } - const parent = parts[index - 1] ?? ""; - return part === "openclaw-cli-images" && /^openclaw(?:-\d+)?$/.test(parent); - }); -} +type PromptImageEntry = { + image: ImageContent; + factIndex: ImageFactIndex; +}; -/** - * Rebuilds the model image array in the same order the prompt saw them: - * existing inline images and offloaded attachments follow `imageOrder`, then - * explicit prompt path/media refs are appended after attachment-owned images. - */ function mergePromptAttachmentImages(params: { imageOrder?: PromptImageOrderEntry[]; + mediaImageLayout?: MediaImageLayout; existingImages?: ImageContent[]; - offloadedImages?: Array; + existingImageFactIndexes?: readonly ImageFactIndex[]; + offloadedImages?: Array; promptRefImages?: ImageContent[]; -}): ImageContent[] { - const promptImages: ImageContent[] = []; - const existingImages = params.existingImages ?? []; +}): PromptImageEntry[] { + const existingImages = (params.existingImages ?? []).map((image, index) => ({ + image, + factIndex: params.existingImageFactIndexes?.[index] ?? null, + })); const offloadedImages = params.offloadedImages ?? []; - - if (params.imageOrder && params.imageOrder.length > 0) { - let inlineIndex = 0; - let offloadedIndex = 0; - for (const entry of params.imageOrder) { - if (entry === "inline") { - const image = existingImages[inlineIndex++]; - if (image) { - promptImages.push(image); - } - continue; - } - const image = offloadedImages[offloadedIndex++]; - if (image) { - promptImages.push(image); - } - } - promptImages.push(...existingImages.slice(inlineIndex)); - while (offloadedIndex < offloadedImages.length) { - const image = offloadedImages[offloadedIndex++]; - if (image) { - promptImages.push(image); - } - } - } else { - promptImages.push(...existingImages); - for (const image of offloadedImages) { - if (image) { - promptImages.push(image); - } - } + const promptRefImages = (params.promptRefImages ?? []).map((image) => ({ + image, + factIndex: null, + })); + const slots: MediaImageLayout["slots"] = + params.mediaImageLayout?.slots ?? params.imageOrder?.map((kind) => ({ kind })) ?? []; + if (slots.length === 0) { + const factOwned = [...offloadedImages, ...existingImages] + .filter((entry): entry is PromptImageEntry => entry !== null && entry.factIndex !== null) + .toSorted((left, right) => (left.factIndex ?? 0) - (right.factIndex ?? 0)); + return [ + ...factOwned, + ...existingImages.filter((entry) => entry.factIndex === null), + ...promptRefImages, + ]; } - promptImages.push(...(params.promptRefImages ?? [])); - return promptImages; + const unusedExisting = [...existingImages]; + const takeExisting = (factIndex: number | null | undefined): PromptImageEntry | undefined => { + const matchIndex = + factIndex === undefined + ? 0 + : unusedExisting.findIndex((entry) => entry.factIndex === factIndex); + if (matchIndex < 0) { + return undefined; + } + return unusedExisting.splice(matchIndex, 1)[0]; + }; + let offloadedIndex = 0; + const ordered = slots.flatMap((slot) => { + const offloaded = slot.kind === "offloaded" ? offloadedImages[offloadedIndex++] : undefined; + const exactExisting = + slot.factIndex !== undefined + ? takeExisting(slot.factIndex) + : slot.kind === "inline" + ? takeExisting(undefined) + : undefined; + const existing = + exactExisting ?? + (slot.kind === "inline" && slot.factIndex !== undefined ? takeExisting(null) : undefined); + if (existing) { + return [existing]; + } + if (slot.kind === "inline") { + return []; + } + return offloaded ? [offloaded] : []; + }); + return [ + ...ordered, + ...unusedExisting, + ...offloadedImages + .slice(offloadedIndex) + .filter((entry): entry is PromptImageEntry => entry !== null), + ...promptRefImages, + ]; } -function createRefCountMap(refs: DetectedImageRef[]): Map { - const counts = new Map(); - for (const ref of refs) { - const key = `${ref.type}\0${normalizeRefForDedupe(ref.resolved)}`; - counts.set(key, (counts.get(key) ?? 0) + 1); - } - return counts; -} - -function consumeRefCount(counts: Map, ref: DetectedImageRef): boolean { - const key = `${ref.type}\0${normalizeRefForDedupe(ref.resolved)}`; - const count = counts.get(key) ?? 0; - if (count <= 0) { - return false; - } - if (count === 1) { - counts.delete(key); - } else { - counts.set(key, count - 1); - } - return true; -} - -/** - * Reads only the leading attachment boilerplate block. User-authored image refs - * after the first blank/non-attachment line must remain prompt refs. - */ -function extractLeadingAttachmentPrompt(prompt: string): string { - const lines = prompt.split(/\r?\n/); - const attachmentLines: string[] = []; - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed) { - break; - } - if (/^\[media attached:\s*\d+\s+files?\]$/i.test(trimmed)) { - attachmentLines.push(trimmed); - continue; - } - if (/^\[media attached(?:\s+\d+\/\d+)?:\s*[^\]]+\]$/i.test(trimmed)) { - attachmentLines.push(trimmed); - continue; - } - break; - } - return attachmentLines.join("\n"); -} - -function extractLeadingInlineAttachmentRefs(prompt: string, count: number): DetectedImageRef[] { - if (count <= 0) { - return []; - } - const attachmentPrompt = extractLeadingAttachmentPrompt(prompt); - if (!attachmentPrompt) { - return []; - } - return detectImageReferences(attachmentPrompt).slice(0, count); -} - -/** - * Finds trailing media:// attachment lines produced by claim-check offload. The - * reverse scan stops at the first non-attachment line so prompt text above it is - * not accidentally treated as attachment boilerplate. - */ -function extractTrailingAttachmentMediaUris(prompt: string, count: number): string[] { - if (count <= 0) { - return []; - } - - const lines = prompt.split(/\r?\n/); - const uris: string[] = []; - for (let index = lines.length - 1; index >= 0 && uris.length < count; index--) { - const line = lines[index]?.trim(); - if (!line || line.includes("\0")) { - break; - } - const match = line.match(/^\[media attached:\s*(media:\/\/inbound\/[^\]\s/\\]+)\]$/); - if (!match?.[1]) { - break; - } - uris.push(match[1]); - } - for (let left = 0, right = uris.length - 1; left < right; left += 1, right -= 1) { - const leftUri = uris.at(left); - const rightUri = uris.at(right); - if (leftUri === undefined || rightUri === undefined) { - break; - } - uris[left] = rightUri; - uris[right] = leftUri; - } - return uris; -} - -/** - * Separates image refs that came from attachment boilerplate from refs the user - * actually typed into the prompt. Attachment refs are already represented by - * existing/offloaded image content and should not be loaded a second time. - */ -function splitPromptAndAttachmentRefs(params: { - prompt: string; - refs: DetectedImageRef[]; - imageOrder?: PromptImageOrderEntry[]; - existingImageCount?: number; -}): { - promptRefs: DetectedImageRef[]; - attachmentRefs: DetectedImageRef[]; -} { - const existingImageCount = params.existingImageCount ?? 0; - const inlineOrderCount = params.imageOrder?.filter((entry) => entry === "inline").length; - // Inline attachments appear at the front of the prompt and are already present in existingImages. - const inlineAttachmentRefCount = Math.min( - existingImageCount, - inlineOrderCount ?? existingImageCount, - ); - const inlineAttachmentRefs = createRefCountMap( - extractLeadingInlineAttachmentRefs(params.prompt, inlineAttachmentRefCount), - ); - const offloadedCount = params.imageOrder?.filter((entry) => entry === "offloaded").length ?? 0; - // Offloaded claim-check attachments are appended after the prompt and loaded through media://. - const attachmentUris = new Set( - offloadedCount > 0 ? extractTrailingAttachmentMediaUris(params.prompt, offloadedCount) : [], - ); - - const promptRefs: DetectedImageRef[] = []; - const attachmentRefs: DetectedImageRef[] = []; - for (const ref of params.refs) { - if (consumeRefCount(inlineAttachmentRefs, ref)) { - continue; - } - if (ref.type === "media-uri" && attachmentUris.has(ref.resolved)) { - attachmentRefs.push(ref); - continue; - } - promptRefs.push(ref); - } - return { promptRefs, attachmentRefs }; -} - -async function sanitizeImagesWithLog( - images: ImageContent[], +async function sanitizeImageEntriesWithLog( + entries: PromptImageEntry[], label: string, imageSanitization?: ImageSanitizationLimits, -): Promise { - const { images: sanitized, dropped } = await sanitizeImageBlocks( - images, - label, - imageSanitization, - ); +): Promise<{ entries: PromptImageEntry[]; failedMediaCount: number }> { + const sanitized: PromptImageEntry[] = []; + let dropped = 0; + let failedMediaCount = 0; + for (const entry of entries) { + const result = await sanitizeImageBlocks([entry.image], label, imageSanitization); + const image = result.images[0]; + if (image) { + sanitized.push({ image, factIndex: entry.factIndex }); + } + dropped += result.dropped; + if (result.dropped > 0 && entry.factIndex !== null) { + failedMediaCount++; + } + } if (dropped > 0) { log.warn(`Native image: dropped ${dropped} image(s) after sanitization (${label}).`); } - return sanitized; + return { entries: sanitized, failedMediaCount }; } -/** - * Detects image references in a user prompt. - * - * Patterns detected: - * - Absolute paths: /path/to/image.png - * - Relative paths: ./image.png, ../images/photo.jpg - * - Home paths: ~/Pictures/screenshot.png - * - file:// URLs: file:///path/to/image.png - * - Message attachments: [Image: source: /path/to/image.jpg] - * - Gateway claim-check URIs: [media attached: media://inbound/] - * - * @param prompt The user prompt text to scan - * @returns Array of detected image references - */ +/** Detects explicit local image paths and file URLs in user prompt text. */ export function detectImageReferences(prompt: string): DetectedImageRef[] { const refs: DetectedImageRef[] = []; const seen = new Set(); + const pathPrompt = prompt.replace(LEGACY_ATTACHMENT_MARKER_PATTERN, (marker) => + " ".repeat(marker.length), + ); - // Dedupe by the user-visible token before resolving so repeated refs keep their first spelling. const addPathRef = (raw: string) => { const trimmed = raw.trim(); const dedupeKey = normalizeRefForDedupe(trimmed); @@ -365,69 +226,17 @@ export function detectImageReferences(prompt: string): DetectedImageRef[] { refs.push({ raw: trimmed, type: "path", resolved }); }; - // Pattern for [media attached: path (type) | url] or [media attached N/M: path (type) | url] format - // Each bracket = ONE file. The | separates path from URL, not multiple files. - // Multi-file format uses separate brackets on separate lines. - MEDIA_ATTACHED_PATTERN.lastIndex = 0; - MESSAGE_IMAGE_PATTERN.lastIndex = 0; FILE_URL_PATTERN.lastIndex = 0; WINDOWS_DRIVE_PATH_PATTERN.lastIndex = 0; PATH_PATTERN.lastIndex = 0; let match: RegExpExecArray | null; - while ((match = MEDIA_ATTACHED_PATTERN.exec(prompt)) !== null) { - const content = match[1]; - if (content === undefined) { - continue; - } - // Skip "[media attached: N files]" header lines - if (/^\d+\s+files?$/i.test(content.trim())) { - continue; - } - - // Check for a Gateway claim-check URI first (media://inbound/). - // This must be tested before the extension-based path regex because the - // URI has no file extension suffix in its base form. - const mediaUriMatch = content.match(MEDIA_URI_REGEX); - const mediaId = mediaUriMatch?.at(1); - if (mediaId && !mediaId.includes("\0")) { - const uri = `media://inbound/${mediaId}`; - const dedupeKey = normalizeRefForDedupe(uri); - if (!seen.has(dedupeKey)) { - seen.add(dedupeKey); - refs.push({ raw: uri, type: "media-uri", resolved: uri }); - } - continue; - } - - // Extract path before the (mime/type) or | delimiter - // Format is: path (type) | url OR just: path (type) - // Path may contain spaces (e.g., "ChatGPT Image Apr 21.png") - // Use non-greedy .+? to stop at first image extension - const pathMatch = content.match(MEDIA_ATTACHED_PATH_PATTERN); - if (pathMatch?.[1]) { - addPathRef(pathMatch[1].trim()); - } - } - - // Pattern for [Image: source: /path/...] format from messaging systems - while ((match = MESSAGE_IMAGE_PATTERN.exec(prompt)) !== null) { - const raw = match[1]?.trim(); - if (raw) { - addPathRef(raw); - } - } - - // Remote HTTP(S) URLs are intentionally ignored. Native image injection is local-only. - - // Pattern for file:// URLs - treat as paths since loadWebMedia handles them - while ((match = FILE_URL_PATTERN.exec(prompt)) !== null) { + while ((match = FILE_URL_PATTERN.exec(pathPrompt)) !== null) { const raw = match[0]; const dedupeKey = normalizeRefForDedupe(raw); if (seen.has(dedupeKey)) { continue; } - // Use fileURLToPath for proper handling (e.g., file://localhost/path) try { const resolved = safeFileURLToPath(raw); if (isOpenClawCliImageCachePath(resolved)) { @@ -436,25 +245,17 @@ export function detectImageReferences(prompt: string): DetectedImageRef[] { seen.add(dedupeKey); refs.push({ raw, type: "path", resolved }); } catch { - // Skip malformed file:// URLs + continue; } } - // Pattern for Windows drive paths. - while ((match = WINDOWS_DRIVE_PATH_PATTERN.exec(prompt)) !== null) { + while ((match = WINDOWS_DRIVE_PATH_PATTERN.exec(pathPrompt)) !== null) { if (match[1]) { addPathRef(match[1]); } } - // Pattern for file paths (absolute, relative, or home) - // Matches: - // - /absolute/path/to/file.ext (including paths with special chars like Messages/Attachments) - // - ./relative/path.ext - // - ../parent/path.ext - // - ~/home/path.ext - while ((match = PATH_PATTERN.exec(prompt)) !== null) { - // Use capture group 1 (the path without delimiter prefix); skip if undefined + while ((match = PATH_PATTERN.exec(pathPrompt)) !== null) { if (match[1]) { addPathRef(match[1]); } @@ -463,13 +264,23 @@ export function detectImageReferences(prompt: string): DetectedImageRef[] { return refs; } -/** - * Resolves and loads one detected image ref into model-ready image content. - * Sandbox refs must validate through the bridge; non-sandbox refs can resolve - * media claim-checks and workspace-relative paths before loadWebMedia enforces - * local-root and size limits. - */ -export async function loadImageFromRef( +function refDedupeKey(ref: DetectedImageRef, workspaceDir?: string): string { + const resolved = + ref.type === "path" && workspaceDir && !path.isAbsolute(ref.resolved) + ? path.resolve(workspaceDir, ref.resolved) + : ref.resolved; + return `${ref.type}\0${normalizeRefForDedupe(resolved)}`; +} + +function rawAliasDedupeKey(alias: string): string | undefined { + return path.isAbsolute(alias) || + /^[A-Za-z]:[\\/]/.test(alias) || + /^[a-z][a-z0-9+.-]*:/i.test(alias) + ? normalizeRefForDedupe(alias) + : undefined; +} + +async function loadImageFromRef( ref: DetectedImageRef, workspaceDir: string, options?: { @@ -482,8 +293,6 @@ export async function loadImageFromRef( try { let targetPath = ref.resolved; - // media:// claim-check refs are resolved only outside sandbox mode; sandbox - // mode validates through resolveSandboxedBridgeMediaPath instead. if (!options?.sandbox) { targetPath = await resolveMediaReferenceLocalPath(targetPath); } @@ -510,7 +319,6 @@ export async function loadImageFromRef( targetPath = path.resolve(workspaceDir, targetPath); } - // loadWebMedia handles local file paths and file:// URLs after the path policy above. const media = options?.sandbox ? await loadWebMedia(targetPath, { maxBytes: options.maxBytes, @@ -519,7 +327,7 @@ export async function loadImageFromRef( }) : await loadWebMedia( targetPath, - options?.workspaceOnly + options?.workspaceOnly || options?.localRoots ? { maxBytes: options.maxBytes, localRoots: options.localRoots ?? [workspaceDir] } : options?.maxBytes, ); @@ -529,137 +337,343 @@ export async function loadImageFromRef( return null; } - // EXIF orientation is already normalized by loadWebMedia -> resizeToJpeg - // Default to JPEG since optimization converts images to JPEG format const mimeType = media.contentType ?? "image/jpeg"; const data = media.buffer.toString("base64"); return { type: "image", data, mimeType }; } catch (err) { - // Log the actual error for debugging (size limits, network failures, etc.) log.debug(`Native image: failed to load ${ref.resolved}: ${formatErrorMessage(err)}`); return null; } } -/** Returns whether the resolved model advertises native image input support. */ function modelSupportsImages(model: { input?: string[] }): boolean { return model.input?.includes("image") ?? false; } -/** - * Detects, loads, orders, and sanitizes the image payload for one prompt turn. - * Attachment boilerplate is separated from user-authored refs so existing - * inline images and offloaded claim-check images are not loaded twice. - */ export async function detectAndLoadPromptImages(params: { prompt: string; + media?: readonly MediaFact[]; workspaceDir: string; model: { input?: string[] }; existingImages?: ImageContent[]; + existingImageFactIndexes?: readonly ImageFactIndex[]; imageOrder?: PromptImageOrderEntry[]; + mediaImageLayout?: MediaImageLayout; maxBytes?: number; maxDimensionPx?: number; workspaceOnly?: boolean; localRoots?: readonly string[]; sandbox?: { root: string; bridge: SandboxFsBridge }; }): Promise<{ - /** Images for the current prompt (existingImages + detected in current prompt) */ images: ImageContent[]; + imageFactIndexes: ImageFactIndex[]; detectedRefs: DetectedImageRef[]; + failedMediaCount: number; loadedCount: number; skippedCount: number; }> { if (!modelSupportsImages(params.model)) { return { images: [], + imageFactIndexes: [], detectedRefs: [], + failedMediaCount: 0, loadedCount: 0, skippedCount: 0, }; } - const allRefs = detectImageReferences(params.prompt); - - if (allRefs.length === 0) { - const sanitizedExistingImages = await sanitizeImagesWithLog( - params.existingImages ?? [], - "prompt:images", - { maxDimensionPx: params.maxDimensionPx }, - ); + const allMediaRefs = collectMediaImageRefs(params.media); + const suppressedFactIndexes = new Set(params.mediaImageLayout?.suppressedFactIndexes ?? []); + for (const ref of allMediaRefs) { + if (!ref || !suppressedFactIndexes.has(ref.factIndex)) { + continue; + } + ref.hydrate = false; + } + const orderRefs = allMediaRefs.filter( + (ref) => !ref || (!suppressedFactIndexes.has(ref.factIndex) && ref.hydrate), + ); + const imageOrder = params.mediaImageLayout?.slots.map((slot) => slot.kind) ?? params.imageOrder; + const refsByFactIndex = new Map( + allMediaRefs.flatMap((ref) => (ref ? [[ref.factIndex, ref] as const] : [])), + ); + // imageOrder describes only images still requiring native delivery; described + // (suppressed) facts must not count against it, or the inference silently + // skips and inline sanitization failures dispatch as success. + const unsuppressedImageFactIndexes = normalizeMediaFacts(params.media).flatMap( + (fact, factIndex) => + isImageMediaFact(fact) && + fact.hydrationSuppressed !== true && + !suppressedFactIndexes.has(factIndex) + ? [factIndex] + : [], + ); + const inferredExistingImageFactIndexes = + imageOrder && unsuppressedImageFactIndexes.length === imageOrder.length + ? imageOrder.flatMap((entry, index) => + entry === "inline" ? [unsuppressedImageFactIndexes[index] ?? null] : [], + ) + : undefined; + const inferredMediaImageLayout = + !params.mediaImageLayout && + imageOrder && + unsuppressedImageFactIndexes.length === imageOrder.length + ? { + slots: imageOrder.map((kind, index) => ({ + kind, + factIndex: unsuppressedImageFactIndexes[index], + })), + suppressedFactIndexes: [], + } + : undefined; + const layoutInlineFactIndexes = resolveLayoutInlineFactIndexes( + params.mediaImageLayout, + params.existingImages?.length ?? 0, + ); + const existingImageFactIndexes = + params.existingImageFactIndexes ?? + layoutInlineFactIndexes ?? + (inferredExistingImageFactIndexes?.length === (params.existingImages?.length ?? 0) + ? inferredExistingImageFactIndexes + : undefined); + const missingInlineMediaCount = countMissingLayoutInlineSlots( + params.mediaImageLayout, + existingImageFactIndexes, + params.existingImages?.length ?? 0, + ); + const attachmentRefs = params.mediaImageLayout + ? params.mediaImageLayout.slots.flatMap((slot) => + slot.kind === "offloaded" + ? [ + { + factIndex: slot.factIndex, + ref: slot.factIndex === undefined ? undefined : refsByFactIndex.get(slot.factIndex), + }, + ] + : [], + ) + : selectMediaImageRefs({ + refs: orderRefs, + existingImageCount: params.existingImages?.length ?? 0, + imageOrder, + }).map((ref) => ({ factIndex: ref?.factIndex, ref })); + const materializedFactIndexes = new Set( + (existingImageFactIndexes ?? []).filter((entry): entry is number => typeof entry === "number"), + ); + const availableMediaRefs = allMediaRefs.filter((ref): ref is MediaImageRef => ref !== undefined); + const selectedAttachmentRefs = attachmentRefs.flatMap(({ ref }) => (ref ? [ref] : [])); + const attachmentKeys = new Set( + selectedAttachmentRefs.map((ref) => refDedupeKey(ref, ref.workspaceDir ?? params.workspaceDir)), + ); + const attachmentRawKeys = new Set( + selectedAttachmentRefs.flatMap((ref) => + ref.aliases.flatMap((alias) => { + const key = rawAliasDedupeKey(alias); + return key ? [key] : []; + }), + ), + ); + const promptRefs = detectImageReferences(params.prompt).filter( + (ref) => + !attachmentRawKeys.has(rawAliasDedupeKey(ref.raw) ?? "") && + !attachmentKeys.has(refDedupeKey(ref, params.workspaceDir)), + ); + const detectedRefs = [ + ...availableMediaRefs.flatMap(({ detect, hydrate, raw, type, resolved }) => + detect !== false && + (hydrate || (!resolved.startsWith("http://") && !resolved.startsWith("https://"))) + ? [{ raw, type, resolved }] + : [], + ), + ...promptRefs, + ]; + if (attachmentRefs.length === 0 && promptRefs.length === 0) { + const existingImages = params.existingImages ?? []; + const sanitized = existingImages.length + ? await sanitizeImageEntriesWithLog( + existingImages.map((image, index) => ({ + image, + factIndex: existingImageFactIndexes?.[index] ?? null, + })), + "prompt:images", + { + maxBytes: params.maxBytes, + maxDimensionPx: params.maxDimensionPx, + }, + ) + : { entries: [], failedMediaCount: 0 }; + const finalized = finalizeRuntimePromptImages(sanitized.entries); return { - images: sanitizedExistingImages, - detectedRefs: [], + ...finalized, + detectedRefs, + failedMediaCount: missingInlineMediaCount + sanitized.failedMediaCount, loadedCount: 0, skippedCount: 0, }; } - log.debug(`Native image: detected ${allRefs.length} image refs in prompt`); - const { promptRefs, attachmentRefs } = splitPromptAndAttachmentRefs({ - prompt: params.prompt, - refs: allRefs, - imageOrder: params.imageOrder, - existingImageCount: params.existingImages?.length, - }); - const promptRefImages: ImageContent[] = []; - const offloadedImages: Array = []; - + log.debug( + `Native image: prepared ${attachmentRefs.length} attachment ref(s) and ${promptRefs.length} explicit prompt ref(s)`, + ); let loadedCount = 0; + let failedMediaCount = missingInlineMediaCount; let skippedCount = 0; - - for (const ref of promptRefs) { - const image = await loadImageFromRef(ref, params.workspaceDir, { + const loadRef = async ( + ref: DetectedImageRef & { workspaceDir?: string }, + ): Promise => { + const image = await loadImageFromRef(ref, ref.workspaceDir ?? params.workspaceDir, { maxBytes: params.maxBytes, workspaceOnly: params.workspaceOnly, - localRoots: params.localRoots, + localRoots: params.localRoots ?? (params.workspaceOnly ? [params.workspaceDir] : undefined), sandbox: params.sandbox, }); if (image) { - promptRefImages.push(image); loadedCount++; log.debug(`Native image: loaded ${ref.type} ${ref.resolved}`); } else { skippedCount++; } + return image; + }; + const offloadedImages: Array = []; + for (const attachment of attachmentRefs) { + const factIndex = attachment.factIndex; + if (factIndex !== undefined && materializedFactIndexes.has(factIndex)) { + offloadedImages.push(null); + continue; + } + const ref = attachment.ref; + if (!ref) { + failedMediaCount++; + offloadedImages.push(null); + continue; + } + const image = ref.hydrate ? await loadRef(ref) : null; + if (ref.hydrate && !image) { + failedMediaCount++; + } + offloadedImages.push(image ? { image, factIndex: ref.factIndex } : null); } - - for (const ref of attachmentRefs) { - const image = await loadImageFromRef(ref, params.workspaceDir, { - maxBytes: params.maxBytes, - workspaceOnly: params.workspaceOnly, - localRoots: params.localRoots, - sandbox: params.sandbox, - }); - offloadedImages.push(image); + const promptRefImages: ImageContent[] = []; + for (const ref of promptRefs) { + const image = await loadRef(ref); if (image) { - loadedCount++; - log.debug(`Native image: loaded ${ref.type} ${ref.resolved}`); - } else { - skippedCount++; + promptRefImages.push(image); } } const promptImages = mergePromptAttachmentImages({ - imageOrder: params.imageOrder, + imageOrder, + mediaImageLayout: params.mediaImageLayout ?? inferredMediaImageLayout, existingImages: params.existingImages, + existingImageFactIndexes, offloadedImages, promptRefImages, }); - const imageSanitization: ImageSanitizationLimits = { + const sanitizedPromptImages = await sanitizeImageEntriesWithLog(promptImages, "prompt:images", { + maxBytes: params.maxBytes, maxDimensionPx: params.maxDimensionPx, - }; - const sanitizedPromptImages = await sanitizeImagesWithLog( - promptImages, - "prompt:images", - imageSanitization, - ); + }); + const finalized = finalizeRuntimePromptImages(sanitizedPromptImages.entries); return { - images: sanitizedPromptImages, - detectedRefs: allRefs, + ...finalized, + detectedRefs, + failedMediaCount: failedMediaCount + sanitizedPromptImages.failedMediaCount, loadedCount, skippedCount, }; } + +/** Hydrates non-enumerable facts carried by queued user turns before provider replay. */ +export async function hydratePromptMediaMessages( + messages: AgentMessage[], + options: { + workspaceDir: string; + model: { input?: string[] }; + maxBytes?: number; + maxDimensionPx?: number; + workspaceOnly?: boolean; + localRoots?: readonly string[]; + sandbox?: { root: string; bridge: SandboxFsBridge }; + }, +): Promise { + let hydrated: AgentMessage[] | undefined; + for (const [index, message] of messages.entries()) { + if (message.role !== "user") { + continue; + } + const runtimeMedia = readRuntimePromptMediaFacts(message); + const media = + runtimeMedia ?? + resolveMediaFacts(message as unknown as Parameters[0]); + const meta = (message as unknown as Record)["__openclaw"]; + const resolvedMedia = runtimeMedia ?? readPersistedPromptMediaFacts(message) ?? media; + const runtimeImageOrder = readRuntimePromptImageOrder(message); + const mediaImageLayout = readPersistedMediaImageLayout(message); + if (!resolvedMedia.length) { + continue; + } + const content = Array.isArray(message.content) + ? message.content + : [{ type: "text" as const, text: message.content }]; + const existingImages = content.filter((block): block is ImageContent => block.type === "image"); + const persistedImageFactIndexes = readPersistedImageBlockFactIndexes(message); + const inlineLayoutFactIndexes = mediaImageLayout?.slots.flatMap((slot) => + slot.kind === "inline" ? [slot.factIndex ?? null] : [], + ); + // Pre-carrier transcripts had no explicit block provenance. Their native + // image blocks were positionally aligned with the first image facts. + const legacyImageFactIndexes = + runtimeMedia === undefined && mediaImageLayout === undefined + ? collectIdentitylessMediaImageFactIndexes(resolvedMedia) + : undefined; + const existingImageFactIndexes = + persistedImageFactIndexes ?? + (inlineLayoutFactIndexes?.length === existingImages.length + ? inlineLayoutFactIndexes + : legacyImageFactIndexes?.slice(0, existingImages.length)); + const result = await detectAndLoadPromptImages({ + prompt: "", + media: resolvedMedia, + workspaceDir: options.workspaceDir, + model: options.model, + existingImages, + existingImageFactIndexes, + imageOrder: runtimeImageOrder, + mediaImageLayout, + maxBytes: options.maxBytes, + maxDimensionPx: options.maxDimensionPx, + workspaceOnly: options.workspaceOnly, + localRoots: options.localRoots, + sandbox: options.sandbox, + }); + const nextMeta = + meta && typeof meta === "object" && !Array.isArray(meta) + ? { ...(meta as Record) } + : {}; + if (result.images.length > 0) { + nextMeta.mediaImageBlockFactIndexes = result.imageFactIndexes; + } else { + delete nextMeta.mediaImageBlockFactIndexes; + } + hydrated ??= messages.slice(); + const hydratedMessage = { + ...message, + content: [...content.filter((block) => block.type !== "image"), ...result.images], + } as AgentMessage; + if (Object.keys(nextMeta).length > 0) { + (hydratedMessage as unknown as Record)["__openclaw"] = nextMeta; + } else { + delete (hydratedMessage as unknown as Record)["__openclaw"]; + } + if (runtimeMedia) { + attachRuntimePromptMediaFacts(hydratedMessage, runtimeMedia, runtimeImageOrder); + } + hydrated[index] = hydratedMessage; + } + return hydrated ?? messages; +} diff --git a/src/agents/embedded-agent-runner/run/plugin-harness-prompt-images.ts b/src/agents/embedded-agent-runner/run/plugin-harness-prompt-images.ts new file mode 100644 index 000000000000..3f94a5c4bd47 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/plugin-harness-prompt-images.ts @@ -0,0 +1,113 @@ +import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants"; +import { isImageMediaFact, resolveMediaFacts } from "../../../media/media-facts.js"; +import { resolveImageSanitizationLimits } from "../../image-sanitization.js"; +import { resolveAttemptWorkspaceSandbox } from "./attempt-setup.js"; +import { detectAndLoadPromptImages } from "./images.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import { + readPersistedMediaImageLayout, + readPersistedPromptMediaFacts, +} from "./prompt-image-metadata.js"; +import type { EmbeddedRunAttemptParams } from "./types.js"; + +function toTypeOnlyImageFact( + fact: NonNullable[number], + hydrationSuppressed: boolean, +): NonNullable[number] { + return { + contentType: fact.contentType, + kind: fact.kind === "sticker" ? "sticker" : "image", + messageId: fact.messageId, + transcribed: fact.transcribed, + ...(fact.hydrationSuppressed === true || hydrationSuppressed + ? { hydrationSuppressed: true } + : {}), + }; +} + +/** Materializes fact-carried images before a plugin harness owns transport. */ +export async function preparePluginHarnessPromptImages(params: { + runParams: RunEmbeddedAgentParams; + runtime: { + sessionId: string; + sessionKey?: string; + workspaceDir: string; + model: EmbeddedRunAttemptParams["model"]; + }; + pluginHarnessOwnsTransport: boolean; +}): Promise<{ + images: RunEmbeddedAgentParams["images"]; + imageOrder: RunEmbeddedAgentParams["imageOrder"]; + media: RunEmbeddedAgentParams["media"]; +}> { + const { runParams, runtime } = params; + if (!params.pluginHarnessOwnsTransport) { + return { + images: runParams.images, + imageOrder: runParams.imageOrder, + media: runParams.media, + }; + } + const persistedMessage = + runParams.userTurnTranscriptRecorder?.message ?? + (await runParams.userTurnTranscriptRecorder?.resolveMessage()); + const persistedMedia = persistedMessage + ? (readPersistedPromptMediaFacts(persistedMessage) ?? + resolveMediaFacts(persistedMessage as unknown as Parameters[0])) + : []; + const hydrationMedia = persistedMedia.length > 0 ? persistedMedia : runParams.media; + if (!hydrationMedia?.some(isImageMediaFact)) { + return { + images: runParams.images, + imageOrder: runParams.imageOrder, + media: runParams.media, + }; + } + + const workspace = await resolveAttemptWorkspaceSandbox({ + ...runParams, + cwd: undefined, + sessionId: runtime.sessionId, + sessionKey: runtime.sessionKey, + workspaceDir: runtime.workspaceDir, + }); + const result = await detectAndLoadPromptImages({ + prompt: "", + media: hydrationMedia, + mediaImageLayout: persistedMessage + ? readPersistedMediaImageLayout(persistedMessage) + : undefined, + workspaceDir: workspace.effectiveWorkspace, + model: runtime.model, + existingImages: runParams.images, + imageOrder: runParams.imageOrder, + maxBytes: MAX_IMAGE_BYTES, + maxDimensionPx: resolveImageSanitizationLimits(runParams.config).maxDimensionPx, + localRoots: workspace.effectiveFsWorkspaceOnly + ? [workspace.effectiveWorkspace, workspace.resolvedWorkspace] + : undefined, + workspaceOnly: workspace.effectiveFsWorkspaceOnly, + sandbox: + workspace.sandbox?.enabled && workspace.sandbox.fsBridge + ? { root: workspace.sandbox.workspaceDir, bridge: workspace.sandbox.fsBridge } + : undefined, + }); + if (result.failedMediaCount > 0) { + throw new Error( + `failed to hydrate ${result.failedMediaCount} structured image attachment(s) for plugin harness input`, + ); + } + const materializedFactIndexes = new Set( + result.imageFactIndexes.filter((index): index is number => index !== null), + ); + const retainedMedia = hydrationMedia?.map((fact, factIndex) => + isImageMediaFact(fact) + ? toTypeOnlyImageFact(fact, !materializedFactIndexes.has(factIndex)) + : fact, + ); + return { + images: result.images, + imageOrder: result.images.length > 0 ? result.images.map(() => "inline" as const) : undefined, + media: retainedMedia?.length ? retainedMedia : undefined, + }; +} diff --git a/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts b/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts new file mode 100644 index 000000000000..c234b55d6467 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts @@ -0,0 +1,118 @@ +import { normalizeMediaFacts, type MediaFact } from "../../../media/media-facts.js"; +import type { AgentMessage } from "../../runtime/index.js"; + +export type ImageFactIndex = number | null; + +export type MediaImageLayout = { + slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>; + suppressedFactIndexes: number[]; +}; + +export function resolveLayoutInlineFactIndexes( + layout: MediaImageLayout | undefined, + existingImageCount: number, +): ImageFactIndex[] | undefined { + const factIndexes = layout?.slots.flatMap((slot) => + slot.kind === "inline" ? [slot.factIndex ?? null] : [], + ); + return factIndexes?.length === existingImageCount ? factIndexes : undefined; +} + +export function countMissingLayoutInlineSlots( + layout: MediaImageLayout | undefined, + existingFactIndexes: readonly ImageFactIndex[] | undefined, + existingImageCount: number, +): number { + if (!layout) { + return 0; + } + const available = existingFactIndexes + ? [...existingFactIndexes] + : Array.from({ length: existingImageCount }, () => null); + let missing = 0; + for (const slot of layout.slots) { + if (slot.kind !== "inline") { + continue; + } + const exactIndex = + slot.factIndex === undefined + ? available.length > 0 + ? 0 + : -1 + : available.findIndex((factIndex) => factIndex === slot.factIndex); + const matchIndex = exactIndex >= 0 ? exactIndex : available.indexOf(null); + if (matchIndex >= 0) { + available.splice(matchIndex, 1); + } else { + missing++; + } + } + return missing; +} + +export function readPersistedImageBlockFactIndexes( + message: AgentMessage, +): ImageFactIndex[] | undefined { + const meta = (message as unknown as Record)["__openclaw"]; + const value = + meta && typeof meta === "object" && !Array.isArray(meta) + ? (meta as Record).mediaImageBlockFactIndexes + : undefined; + if (!Array.isArray(value)) { + return undefined; + } + return value.map((entry) => + typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0 ? entry : null, + ); +} + +export function readPersistedPromptMediaFacts(message: AgentMessage): MediaFact[] | undefined { + const meta = (message as unknown as Record)["__openclaw"]; + const media = + meta && typeof meta === "object" && !Array.isArray(meta) + ? (meta as Record).media + : undefined; + return Array.isArray(media) ? normalizeMediaFacts(media as MediaFact[]) : undefined; +} + +export function readPersistedMediaImageLayout(message: AgentMessage): MediaImageLayout | undefined { + const meta = (message as unknown as Record)["__openclaw"]; + if (!meta || typeof meta !== "object" || Array.isArray(meta)) { + return undefined; + } + const layout = (meta as Record).mediaImageLayout; + if (!layout || typeof layout !== "object" || Array.isArray(layout)) { + return undefined; + } + const record = layout as Record; + const slots = Array.isArray(record.slots) + ? record.slots.flatMap((entry) => { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + return []; + } + const slot = entry as Record; + if (slot.kind !== "inline" && slot.kind !== "offloaded") { + return []; + } + const kind: MediaImageLayout["slots"][number]["kind"] = slot.kind; + const factIndex = slot.factIndex; + return [ + { + kind, + ...(typeof factIndex === "number" && Number.isSafeInteger(factIndex) && factIndex >= 0 + ? { factIndex } + : {}), + }, + ]; + }) + : []; + const suppressedFactIndexes = Array.isArray(record.suppressedFactIndexes) + ? record.suppressedFactIndexes.filter( + (entry): entry is number => + typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0, + ) + : []; + return slots.length > 0 || suppressedFactIndexes.length > 0 + ? { slots, suppressedFactIndexes } + : undefined; +} diff --git a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.media.test.ts b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.media.test.ts new file mode 100644 index 000000000000..b779f378bdc1 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.media.test.ts @@ -0,0 +1,303 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js"; +import { readRuntimePromptImageFactIndexes } from "../../../media/runtime-prompt-image-provenance.js"; +import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js"; +import { detectAndLoadPromptImages } from "./images.js"; +import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js"; + +const TINY_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg=="; + +describe("plugin harness prompt media", () => { + it("hydrates plugin images and preserves serialized replay order with non-image facts", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-harness-media-")); + const workspaceDir = path.join(stateDir, "workspace"); + const inboundDir = path.join(stateDir, "media", "inbound"); + const mediaId = "photo.png"; + const imagePath = path.join(inboundDir, mediaId); + await fs.mkdir(workspaceDir, { recursive: true }); + await fs.mkdir(inboundDir, { recursive: true }); + await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64")); + const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + const documentFact = { + path: path.join(workspaceDir, "misleading.png"), + contentType: "application/pdf", + kind: "document" as const, + }; + const input = { + runParams: { + agentId: "main", + config: { agents: { defaults: { sandbox: { mode: "off" } } } }, + imageOrder: ["offloaded"], + media: [documentFact, { url: `media://inbound/${mediaId}`, contentType: "image/png" }], + sessionId: "session-1", + userTurnTranscriptRecorder: { + message: { + role: "user", + content: "inspect", + MediaPaths: [imagePath, documentFact.path], + MediaTypes: ["image/png", "application/pdf"], + __openclaw: { + mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] }, + }, + }, + }, + }, + runtime: { + model: { input: ["text", "image"] }, + sessionId: "session-1", + workspaceDir, + }, + pluginHarnessOwnsTransport: true, + } as unknown as Parameters[0]; + + try { + const result = await preparePluginHarnessPromptImages(input); + + expect(result.images).toEqual([ + { type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }, + ]); + expect(readRuntimePromptImageFactIndexes(result.images ?? [])).toEqual([0]); + expect(result.imageOrder).toEqual(["inline"]); + expect(result.media?.[0]).toMatchObject({ contentType: "image/png", kind: "image" }); + expect(result.media?.[0]).not.toHaveProperty("path"); + expect(result.media?.[0]).not.toHaveProperty("url"); + expect(result.media?.[1]).toMatchObject(documentFact); + + const serialized = JSON.stringify(result); + const restored = JSON.parse(serialized) as typeof result; + const replay = await detectAndLoadPromptImages({ + prompt: "", + media: restored.media, + workspaceDir, + model: { input: ["text", "image"] }, + existingImages: restored.images, + imageOrder: restored.imageOrder, + }); + expect(replay.failedMediaCount).toBe(0); + expect(replay.images).toEqual(result.images); + expect(replay.imageFactIndexes).toEqual([0]); + } finally { + envSnapshot.restore(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + + it("surfaces a failed image hydration before plugin dispatch", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-harness-failed-media-")); + try { + await expect( + preparePluginHarnessPromptImages({ + runParams: { + agentId: "main", + config: { agents: { defaults: { sandbox: { mode: "off" } } } }, + imageOrder: ["offloaded"], + media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }], + sessionId: "session-failed", + }, + runtime: { + model: { input: ["text", "image"] }, + sessionId: "session-failed", + workspaceDir, + }, + pluginHarnessOwnsTransport: true, + } as unknown as Parameters[0]), + ).rejects.toThrow("failed to hydrate 1 structured image attachment"); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + + it("surfaces an unsuppressed identity-less inline fact with no image block", async () => { + await expect( + preparePluginHarnessPromptImages({ + runParams: { + agentId: "main", + config: { agents: { defaults: { sandbox: { mode: "off" } } } }, + imageOrder: ["inline"], + media: [{ kind: "image" }], + sessionId: "session-missing-inline", + }, + runtime: { + model: { input: ["text", "image"] }, + sessionId: "session-missing-inline", + workspaceDir: "/tmp", + }, + pluginHarnessOwnsTransport: true, + } as unknown as Parameters[0]), + ).rejects.toThrow("failed to hydrate 1 structured image attachment"); + }); + + it("surfaces a fact-owned image dropped during host sanitization", async () => { + await expect( + preparePluginHarnessPromptImages({ + runParams: { + agentId: "main", + config: { agents: { defaults: { sandbox: { mode: "off" } } } }, + images: [{ type: "image", data: "%%%", mimeType: "image/png" }], + imageOrder: ["inline"], + media: [{ kind: "image" }], + sessionId: "session-sanitize-failed", + }, + runtime: { + model: { input: ["text", "image"] }, + sessionId: "session-sanitize-failed", + workspaceDir: "/tmp", + }, + pluginHarnessOwnsTransport: true, + } as unknown as Parameters[0]), + ).rejects.toThrow("failed to hydrate 1 structured image attachment"); + }); + + it("surfaces inline sanitization failure when a preceding plugin image fact is suppressed", async () => { + await expect( + preparePluginHarnessPromptImages({ + runParams: { + agentId: "main", + config: { agents: { defaults: { sandbox: { mode: "off" } } } }, + images: [{ type: "image", data: "%%%", mimeType: "image/png" }], + imageOrder: ["inline"], + media: [ + { + path: "/tmp/described-missing.png", + contentType: "image/png", + hydrationSuppressed: true, + }, + { path: "/tmp/inline.png", contentType: "image/png" }, + ], + sessionId: "session-suppressed-before-inline", + }, + runtime: { + model: { input: ["text", "image"] }, + sessionId: "session-suppressed-before-inline", + workspaceDir: "/tmp", + }, + pluginHarnessOwnsTransport: true, + } as unknown as Parameters[0]), + ).rejects.toThrow("failed to hydrate 1 structured image attachment"); + }); + + it("retains an intentionally non-hydrating remote-only image as a type-only fact", async () => { + const media = buildInboundMediaNoteProjection({ + MediaPaths: [""], + MediaUrls: ["https://example.com/described.png"], + MediaTypes: ["image/png"], + MediaUnderstanding: [ + { + kind: "image.description", + attachmentIndex: 0, + text: "already described", + provider: "test", + }, + ], + }).media; + const result = await preparePluginHarnessPromptImages({ + runParams: { + agentId: "main", + config: { agents: { defaults: { sandbox: { mode: "off" } } } }, + media, + sessionId: "session-described", + }, + runtime: { + model: { input: ["text", "image"] }, + sessionId: "session-described", + workspaceDir: "/tmp", + }, + pluginHarnessOwnsTransport: true, + } as unknown as Parameters[0]); + + expect(result.images).toEqual([]); + expect(result.media?.[0]).toMatchObject({ + contentType: "image/png", + kind: "image", + hydrationSuppressed: true, + }); + expect(result.media?.[0]).not.toHaveProperty("path"); + expect(result.media?.[0]).not.toHaveProperty("url"); + }); + + it("retains layout-derived suppression after plugin host materialization", async () => { + const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" }; + const result = await preparePluginHarnessPromptImages({ + runParams: { + agentId: "main", + config: { agents: { defaults: { sandbox: { mode: "off" } } } }, + images: [inlineImage], + imageOrder: ["inline"], + media: [ + { path: "/tmp/described.png", contentType: "image/png" }, + { path: "/tmp/inline.png", contentType: "image/png" }, + ], + sessionId: "session-layout-suppressed", + userTurnTranscriptRecorder: { + message: { + role: "user", + content: "compare", + MediaPaths: ["/tmp/described.png", "/tmp/inline.png"], + MediaTypes: ["image/png", "image/png"], + __openclaw: { + mediaImageLayout: { + slots: [{ kind: "inline", factIndex: 1 }], + suppressedFactIndexes: [0], + }, + }, + }, + }, + }, + runtime: { + model: { input: ["text", "image"] }, + sessionId: "session-layout-suppressed", + workspaceDir: "/tmp", + }, + pluginHarnessOwnsTransport: true, + } as unknown as Parameters[0]); + + expect(result.images).toEqual([inlineImage]); + expect(result.imageOrder).toEqual(["inline"]); + expect(result.media?.[0]).toMatchObject({ kind: "image", hydrationSuppressed: true }); + expect(result.media?.[1]).toMatchObject({ kind: "image" }); + expect(result.media?.[1]).not.toHaveProperty("hydrationSuppressed"); + }); + + it("keeps unsupported native images as aligned type-only facts", async () => { + const media = [ + { path: "/tmp/photo.png", contentType: "image/png" }, + { path: "/tmp/inferred.png", kind: "unknown" as const }, + ]; + const result = await preparePluginHarnessPromptImages({ + runParams: { + agentId: "main", + config: { agents: { defaults: { sandbox: { mode: "off" } } } }, + media, + sessionId: "session-text-only", + }, + runtime: { + model: { input: ["text"] }, + sessionId: "session-text-only", + workspaceDir: "/tmp", + }, + pluginHarnessOwnsTransport: true, + } as unknown as Parameters[0]); + + expect(result.images).toEqual([]); + expect(result.media?.[0]).toMatchObject({ contentType: "image/png" }); + expect(result.media?.[0]).not.toHaveProperty("path"); + expect(result.media?.[1]).toMatchObject({ kind: "image" }); + expect(result.media?.[1]).not.toHaveProperty("path"); + }); + + it("leaves facts untouched when the native harness owns transport", async () => { + const media = [{ path: "/tmp/photo.png", contentType: "image/png" }]; + const result = await preparePluginHarnessPromptImages({ + runParams: { media }, + runtime: {}, + pluginHarnessOwnsTransport: false, + } as unknown as Parameters[0]); + + expect(result).toEqual({ images: undefined, imageOrder: undefined, media }); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts index 7148002f9e12..cf1b3354326c 100644 --- a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts +++ b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts @@ -14,6 +14,7 @@ import { EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, } from "./lane-runtime.js"; import type { RunEmbeddedAgentParams } from "./params.js"; +import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js"; import { resolveSkillWorkshopAttemptParams } from "./skill-workshop-attempt-params.js"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptTrajectoryRecorder } from "./types.js"; @@ -161,6 +162,11 @@ export async function dispatchEmbeddedRunAttempt(input: { }; let cancellationRequested = false; + const promptMedia = await preparePluginHarnessPromptImages({ + runParams: params, + runtime, + pluginHarnessOwnsTransport: control.pluginHarnessOwnsTransport, + }); const attemptParams: EmbeddedRunAttemptParams = { operation: "attempt", sessionId: runtime.sessionId, @@ -223,8 +229,9 @@ export async function dispatchEmbeddedRunAttempt(input: { skipPreparedUserTurnMessage: runtime.skipPreparedUserTurnMessage, currentInboundEventKind: params.currentInboundEventKind, currentInboundContext: params.currentInboundContext, - images: params.images, - imageOrder: params.imageOrder, + images: promptMedia.images, + imageOrder: promptMedia.imageOrder, + media: promptMedia.media, clientTools: params.clientTools, disableTools: params.disableTools, provider: runtime.provider, diff --git a/src/agents/sessions/agent-session-prompting.ts b/src/agents/sessions/agent-session-prompting.ts index a1218bb5b324..5f88e3726245 100644 --- a/src/agents/sessions/agent-session-prompting.ts +++ b/src/agents/sessions/agent-session-prompting.ts @@ -1,6 +1,8 @@ import { readFileSync } from "node:fs"; import type { ImageContent, TextContent } from "../../llm/types.js"; import { attachRuntimePromptMediaFacts, type MediaFact } from "../../media/media-facts.js"; +import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; +import { readRuntimePromptImageFactIndexes } from "../../media/runtime-prompt-image-provenance.js"; import { attachRuntimeUserTurnTranscriptContext } from "../../sessions/user-turn-transcript-runtime-context.js"; import type { PersistedUserTurnMessage, @@ -89,6 +91,21 @@ export abstract class AgentSessionPrompting extends AgentSessionBase { return [{ type: "text", text }, ...(images ?? [])]; } + private createUserMessage(text: string, images?: ImageContent[]): PersistedUserTurnMessage { + const message = { + role: "user", + content: this.createUserContent(text, images), + timestamp: Date.now(), + } satisfies PersistedUserTurnMessage; + const imageFactIndexes = readRuntimePromptImageFactIndexes(images); + return imageFactIndexes + ? ({ + ...message, + __openclaw: { mediaImageBlockFactIndexes: imageFactIndexes }, + } as unknown as PersistedUserTurnMessage) + : message; + } + /** * Send a prompt to the agent. * - Handles extension commands immediately, even during streaming @@ -188,11 +205,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase { messages = []; // Add user message - messages.push({ - role: "user", - content: this.createUserContent(expandedText, currentImages), - timestamp: Date.now(), - }); + messages.push(this.createUserMessage(expandedText, currentImages)); // Inject any pending "nextTurn" messages as context alongside the user message for (const msg of this.pendingNextTurnMessages) { @@ -322,6 +335,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase { images?: ImageContent[], userTurnTranscriptRecorder?: UserTurnTranscriptRecorder, media?: MediaFact[], + imageOrder?: PromptImageOrderEntry[], ): Promise { // Check for extension commands (cannot be queued) if (text.startsWith("/")) { @@ -340,6 +354,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase { ? { message: preparedMessage, recorder: userTurnTranscriptRecorder } : undefined, media, + imageOrder, ); } @@ -374,16 +389,13 @@ export abstract class AgentSessionPrompting extends AgentSessionBase { recorder: UserTurnTranscriptRecorder; }, media?: MediaFact[], + imageOrder?: PromptImageOrderEntry[], ): Promise { this.steeringMessages.push(text); this.emitQueueUpdate(); - const runtimeMessage = { - role: "user", - content: this.createUserContent(text, images), - timestamp: Date.now(), - } satisfies PersistedUserTurnMessage; + const runtimeMessage = this.createUserMessage(text, images); const promptMessage = media?.length - ? attachRuntimePromptMediaFacts(runtimeMessage, media) + ? attachRuntimePromptMediaFacts(runtimeMessage, media, imageOrder) : runtimeMessage; this.agent.steer( transcriptContext diff --git a/src/agents/sessions/sdk.test.ts b/src/agents/sessions/sdk.test.ts index 695509c5d473..e820984b760a 100644 --- a/src/agents/sessions/sdk.test.ts +++ b/src/agents/sessions/sdk.test.ts @@ -4,7 +4,9 @@ import { createAssistantMessageEventStream, type AssistantMessage } from "opencl import { Type } from "typebox"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { getStreamLlmRuntime } from "../../llm/model-runtime-binding.js"; -import type { Model, SimpleStreamOptions } from "../../llm/types.js"; +import type { ImageContent, Model, SimpleStreamOptions } from "../../llm/types.js"; +import { readRuntimePromptImageOrder } from "../../media/media-facts.js"; +import { finalizeRuntimePromptImages } from "../../media/runtime-prompt-image-provenance.js"; import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; @@ -295,8 +297,13 @@ describe("AgentSession queued user turns", () => { const session = await createSessionFromManager(SessionManager.inMemory()); const steer = vi.spyOn(session.agent, "steer").mockImplementation(() => undefined); const media = [{ path: "/tmp/a.png", contentType: "image/png" }]; + const imageOrder = ["inline"] as const; + const image: ImageContent = { type: "image", data: "aW1hZ2U=", mimeType: "image/png" }; + const { images } = finalizeRuntimePromptImages([{ image, factIndex: 0 }]); - await session.steer("[media attached: /tmp/a.png (image/png)]", undefined, undefined, media); + await session.steer("[media attached: /tmp/a.png (image/png)]", images, undefined, media, [ + ...imageOrder, + ]); const runtimeMessage = steer.mock.calls[0]?.[0]; expect(runtimeMessage).toBeDefined(); @@ -304,12 +311,16 @@ describe("AgentSession queued user turns", () => { (symbol) => Symbol.keyFor(symbol) === "openclaw.runtimePromptMediaFacts", ); expect(mediaSymbol).toBeDefined(); - if (!mediaSymbol) { - throw new Error("expected runtime prompt media symbol"); + if (!runtimeMessage || !mediaSymbol) { + throw new Error("expected runtime prompt media message and symbol"); } expect((runtimeMessage as unknown as Record)[mediaSymbol]).toEqual([ expect.objectContaining({ path: "/tmp/a.png", contentType: "image/png", kind: "image" }), ]); + expect(readRuntimePromptImageOrder(runtimeMessage)).toEqual(imageOrder); + expect((runtimeMessage as unknown as Record)["__openclaw"]).toEqual({ + mediaImageBlockFactIndexes: [0], + }); expect(JSON.stringify(runtimeMessage)).not.toContain("runtimePromptMediaFacts"); }); }); diff --git a/src/auto-reply/media-note.test.ts b/src/auto-reply/media-note.test.ts index 710585119d79..5f55fe259b5f 100644 --- a/src/auto-reply/media-note.test.ts +++ b/src/auto-reply/media-note.test.ts @@ -111,7 +111,7 @@ describe("buildInboundMediaNote", () => { }); it("keeps image attachments after image descriptions are added", () => { - const note = buildInboundMediaNote({ + const projection = buildInboundMediaNoteProjection({ MediaPaths: ["/tmp/photo.png"], MediaUrls: ["https://example.com/photo.png"], MediaTypes: ["image/png"], @@ -124,9 +124,20 @@ describe("buildInboundMediaNote", () => { }, ], }); - expect(note).toBe( + expect(projection.text).toBe( "[media attached: /tmp/photo.png (image/png) | https://example.com/photo.png]", ); + expect(projection.media).toEqual([ + { + path: "/tmp/photo.png", + url: "https://example.com/photo.png", + contentType: "image/png", + kind: "image", + transcribed: false, + messageId: undefined, + hydrationSuppressed: true, + }, + ]); }); it("keeps image attachments when image understanding succeeds via decisions", () => { diff --git a/src/auto-reply/media-note.ts b/src/auto-reply/media-note.ts index 72e8a240c60d..e47043fa80e5 100644 --- a/src/auto-reply/media-note.ts +++ b/src/auto-reply/media-note.ts @@ -131,12 +131,20 @@ function collectTranscribedAudioAttachmentIndices( return transcribedAudioIndices; } +function collectDescribedImageAttachmentIndices(ctx: MsgContext): Set { + return new Set( + ctx.MediaUnderstanding?.flatMap((output) => + output.kind === "image.description" ? [output.attachmentIndex] : [], + ) ?? [], + ); +} + type InboundMediaNoteProjection = { text?: string; media: MediaFact[]; }; -/** Formats prompt-visible attachment text and retains the facts represented by it. */ +/** Formats prompt-visible attachment text and retains facts that still need native hydration. */ export function buildInboundMediaNoteProjection(ctx: MsgContext): InboundMediaNoteProjection { // Attachment indices follow MediaPaths/MediaUrls ordering as supplied by the channel. const pathsFromArray = Array.isArray(ctx.MediaPaths) ? ctx.MediaPaths : undefined; @@ -196,14 +204,22 @@ export function buildInboundMediaNoteProjection(ctx: MsgContext): InboundMediaNo return { media: [] }; } const facts = resolveMediaFacts(ctx); + const describedImageIndices = collectDescribedImageAttachmentIndices(ctx); const media = normalizeMediaFacts( - entries.map((entry) => ({ - ...facts[entry.index], - path: entry.path, - url: entry.url, - contentType: entry.type, - })), + entries.map((entry) => + Object.assign({}, facts[entry.index], { + path: entry.path, + url: entry.url, + contentType: entry.type, + }), + ), ); + for (const [position, entry] of entries.entries()) { + const fact = media[position]; + if (fact && describedImageIndices.has(entry.index)) { + fact.hydrationSuppressed = true; + } + } if (entries.length === 1) { return { text: formatMediaAttachedLine({ diff --git a/src/auto-reply/reply/agent-runner.ts b/src/auto-reply/reply/agent-runner.ts index 6eb1bbc5fd75..7a1f4bd6cce7 100644 --- a/src/auto-reply/reply/agent-runner.ts +++ b/src/auto-reply/reply/agent-runner.ts @@ -1419,6 +1419,7 @@ export async function runReplyAgent(params: { steeringMode: "all", isInboundUserMessage: true, ...(followupRun.images?.length ? { images: followupRun.images } : {}), + ...(followupRun.imageOrder?.length ? { imageOrder: followupRun.imageOrder } : {}), ...(followupRun.media?.length ? { media: followupRun.media } : {}), ...(turnAdoptionLifecycle ? { waitForTranscriptCommit: true } : {}), ...(resolvedQueue.debounceMs !== undefined ? { debounceMs: resolvedQueue.debounceMs } : {}), diff --git a/src/auto-reply/reply/current-turn-images.ts b/src/auto-reply/reply/current-turn-images.ts index a5ced0176a46..7a274cd35121 100644 --- a/src/auto-reply/reply/current-turn-images.ts +++ b/src/auto-reply/reply/current-turn-images.ts @@ -137,6 +137,7 @@ function appendOrderedImages(params: { function resolveMergedTurnImages(entries: OrderedTurnImage[]): { images?: ImageContent[]; imageOrder?: PromptImageOrderEntry[]; + imageSourceIndexes?: Array; } { if (entries.length === 0) { return {}; @@ -151,10 +152,14 @@ function resolveMergedTurnImages(entries: OrderedTurnImage[]): { return left.sequence - right.sequence; }); const images = merged.flatMap((entry) => (entry.image ? [entry.image] : [])); - return { + const result = { ...(images.length > 0 ? { images } : {}), imageOrder: merged.map((entry) => entry.imageOrder), }; + Object.defineProperty(result, "imageSourceIndexes", { + value: merged.map((entry) => entry.sourceIndex), + }); + return result; } /** Resolves current-turn image attachments that were not already described by media understanding. */ @@ -167,6 +172,7 @@ export async function resolveCurrentTurnImages(params: { }): Promise<{ images?: ImageContent[]; imageOrder?: PromptImageOrderEntry[]; + imageSourceIndexes?: Array; }> { const entries: OrderedTurnImage[] = []; appendOrderedImages({ diff --git a/src/auto-reply/reply/get-reply-run.media-only.test.ts b/src/auto-reply/reply/get-reply-run.media-only.test.ts index 824c1de14061..6a5acc4022a5 100644 --- a/src/auto-reply/reply/get-reply-run.media-only.test.ts +++ b/src/auto-reply/reply/get-reply-run.media-only.test.ts @@ -1399,6 +1399,13 @@ describe("runPreparedReply media-only handling", () => { expect(call.followupRun.images).toBeUndefined(); expect(call.followupRun.imageOrder).toBeUndefined(); expect(call.followupRun.prompt).toContain("a tiny dot image"); + expect( + ( + call.followupRun.userTurnTranscriptRecorder?.message as unknown as Record + )?.["__openclaw"], + ).toMatchObject({ + mediaImageLayout: { slots: [], suppressedFactIndexes: [0, 1] }, + }); }); it("rehydrates only current MediaPaths missing image understanding", async () => { @@ -1462,6 +1469,16 @@ describe("runPreparedReply media-only handling", () => { mimeType: "image/png", }, ]); + expect( + ( + call.followupRun.userTurnTranscriptRecorder?.message as unknown as Record + )?.["__openclaw"], + ).toMatchObject({ + mediaImageLayout: { + slots: [{ kind: "inline", factIndex: 1 }], + suppressedFactIndexes: [0], + }, + }); expect(call.followupRun.imageOrder).toEqual(["inline"]); expect(call.followupRun.prompt).toContain("a tiny dot image"); }); diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index 6193a730803d..0ee7c96b1b5e 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -43,7 +43,7 @@ import { measureDiagnosticsTimelineSpan } from "../../infra/diagnostics-timeline import { isFastTestRuntimeEnv } from "../../infra/env.js"; import { resolveHeartbeatRunScope } from "../../infra/heartbeat-run-scope.js"; import type { ExtractedFileImage } from "../../media-understanding/extracted-file-images.js"; -import { resolveMediaFacts, type MediaFact } from "../../media/media-facts.js"; +import { isImageMediaFact, resolveMediaFacts, type MediaFact } from "../../media/media-facts.js"; import { clearCommandLane, getQueueSize } from "../../process/command-queue.js"; import { isAcpSessionKey, @@ -54,6 +54,7 @@ import { MEDIA_ONLY_USER_TEXT } from "../../sessions/user-turn-media.js"; import { createUserTurnTranscriptRecorder, resolvePersistedUserTurnText, + type UserTurnInput, } from "../../sessions/user-turn-transcript.js"; import { createLazyImportLoader } from "../../shared/lazy-promise.js"; import type { SilentReplyConversationType } from "../../shared/silent-reply-policy.js"; @@ -162,6 +163,73 @@ type InternalGetReplyOptions = BaseInternalGetReplyOptions & { type AgentDefaults = NonNullable["defaults"]; type ExecOverrides = Pick; const EPOCH_MILLISECONDS_THRESHOLD = 1_000_000_000_000; +function buildPersistedMediaImageLayout(params: { + ctx: MsgContext; + media: readonly MediaFact[]; + ctxMediaCount: number; + imageOrder?: readonly ("inline" | "offloaded")[]; + imageSourceIndexes?: readonly (number | undefined)[]; +}): NonNullable | undefined { + const describedAttachmentIndexes = new Set( + params.ctx.MediaUnderstanding?.flatMap((output) => + output.kind === "image.description" ? [output.attachmentIndex] : [], + ) ?? [], + ); + const suppressedFactIndexes: number[] = []; + const imageFactIndexes: number[] = []; + for (const [factIndex, fact] of params.media.entries()) { + if (!isImageMediaFact(fact)) { + continue; + } + imageFactIndexes.push(factIndex); + if ( + (factIndex < params.ctxMediaCount && describedAttachmentIndexes.has(factIndex)) || + fact.hydrationSuppressed === true + ) { + suppressedFactIndexes.push(factIndex); + } + } + if (imageFactIndexes.length === 0) { + return undefined; + } + const suppressed = new Set(suppressedFactIndexes); + const used = new Set(); + const unsuppressedFactCount = imageFactIndexes.filter((index) => !suppressed.has(index)).length; + const canInferByPosition = unsuppressedFactCount === (params.imageOrder?.length ?? 0); + const takeNextFactIndex = (): number | undefined => + imageFactIndexes.find((index) => !suppressed.has(index) && !used.has(index)); + const slots = (params.imageOrder ?? []).map((kind, index) => { + const sourceIndex = params.imageSourceIndexes?.[index]; + const sourceFact = sourceIndex === undefined ? undefined : params.media[sourceIndex]; + const factIndex = + sourceIndex !== undefined + ? sourceFact && + isImageMediaFact(sourceFact) && + !suppressed.has(sourceIndex) && + !used.has(sourceIndex) + ? sourceIndex + : undefined + : canInferByPosition + ? takeNextFactIndex() + : undefined; + if (factIndex !== undefined) { + used.add(factIndex); + } + return factIndex === undefined ? { kind } : { kind, factIndex }; + }); + for (const factIndex of imageFactIndexes) { + if (!suppressed.has(factIndex) && !used.has(factIndex)) { + slots.push({ kind: "offloaded", factIndex }); + } + } + if (slots.length === 0 && suppressedFactIndexes.length === 0) { + return undefined; + } + return { + slots, + ...(suppressedFactIndexes.length > 0 ? { suppressedFactIndexes } : {}), + }; +} function hasResolvedThinkingCatalogEntry(params: { catalog?: readonly ThinkingCatalogEntry[]; @@ -1466,7 +1534,15 @@ export async function runPreparedReply( : undefined); setChannelSourceTurnId(sessionCtx, sourceTurnId); const persistGroupSender = replyRoute.chatType === "group" || replyRoute.chatType === "channel"; - const userTurnMediaForPersistence = [...resolveMediaFacts(ctx), ...(opts?.media ?? [])]; + const ctxMediaForPersistence = resolveMediaFacts(ctx); + const userTurnMediaForPersistence = [...ctxMediaForPersistence, ...(opts?.media ?? [])]; + const mediaImageLayout = buildPersistedMediaImageLayout({ + ctx, + media: userTurnMediaForPersistence, + ctxMediaCount: ctxMediaForPersistence.length, + imageOrder: currentTurnImages.imageOrder, + imageSourceIndexes: currentTurnImages.imageSourceIndexes, + }); const inputProvenance = ctx.InputProvenance ?? sessionCtx.InputProvenance; const userTurnTimestamp = normalizeMessageTimestampMs(ctx.Timestamp); // prompt-prelude substitutes MEDIA_ONLY_USER_TEXT as transcriptBody for @@ -1518,6 +1594,7 @@ export async function runPreparedReply( : {}), ...(transport ? { transport } : {}), ...(userTurnMediaForPersistence.length > 0 ? { media: userTurnMediaForPersistence } : {}), + ...(mediaImageLayout ? { mediaImageLayout } : {}), // Persist the message's own arrival timestamp so the single // LLM-boundary stamping site (normalizeMessagesForLlmBoundary) can // derive a stable per-message `[DOW YYYY-MM-DD HH:MM TZ]` prefix that diff --git a/src/auto-reply/reply/reply-run-registry.ts b/src/auto-reply/reply/reply-run-registry.ts index 277c594e44fd..878d22c125db 100644 --- a/src/auto-reply/reply/reply-run-registry.ts +++ b/src/auto-reply/reply/reply-run-registry.ts @@ -18,6 +18,7 @@ import { } from "../../logging/diagnostic-run-activity.js"; import { diagnosticLogger as diag } from "../../logging/diagnostic-runtime.js"; import type { MediaFact } from "../../media/media-facts.js"; +import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.types.js"; import { resolveGlobalSingleton } from "../../shared/global-singleton.js"; import { resolveTimerTimeoutMs } from "../../shared/number-coercion.js"; @@ -41,6 +42,7 @@ export type ReplyBackendQueueMessageOptions = { debounceMs?: number; /** Ordered current-turn images to inject with the steering text. */ images?: ImageContent[]; + imageOrder?: PromptImageOrderEntry[]; /** Ordered facts represented by attachment text in this steering prompt. */ media?: MediaFact[]; deliveryTimeoutMs?: number; diff --git a/src/gateway/chat-attachments.test.ts b/src/gateway/chat-attachments.test.ts index 5ac3482d8379..d84eb4e63d29 100644 --- a/src/gateway/chat-attachments.test.ts +++ b/src/gateway/chat-attachments.test.ts @@ -232,10 +232,8 @@ describe("parseMessageWithAttachments", () => { expect(ref.mimeType).toBe("application/pdf"); expect(ref.label).toBe("report.pdf"); expect(ref.mediaRef).toMatch(/^media:\/\/inbound\//); - // Non-image offloads MUST NOT inject a media://URI into the message — - // the caller is responsible for routing offloadedRefs[].path into - // ctx.MediaPaths so the workspace stage surfaces a real path. - expect(parsed.message).toBe("read this"); + expect(parsed.message).toBe(`read this\n[media attached: ${ref.mediaRef}]`); + expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message); expect(saveMediaBufferMock).toHaveBeenCalledOnce(); expect(savedMime()).toBe("application/pdf"); expect(logs).toHaveLength(0); @@ -251,7 +249,10 @@ describe("parseMessageWithAttachments", () => { expect(parsed.offloadedRefs).toHaveLength(1); expect(parsed.offloadedRefs[0]?.mimeType).toBe("application/octet-stream"); expect(savedMime()).toBe("application/octet-stream"); - expect(parsed.message).toBe("take a look"); + expect(parsed.message).toBe( + `take a look\n[media attached: ${parsed.offloadedRefs[0]?.mediaRef}]`, + ); + expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message); expect(logs).toHaveLength(0); }); @@ -272,15 +273,12 @@ describe("parseMessageWithAttachments", () => { expect(parsed.imageOrder).toEqual(["inline"]); }); - it("excludes non-image offloads from imageOrder in mixed batches", async () => { + it("keeps mixed image/PDF markers in normal and image-stripped routing order", async () => { // Regression: a prior revision pushed "offloaded" for every offload, // including non-image files. In a [non-image, inline, offloaded-image] // batch that produced imageOrder=["offloaded","inline","offloaded"] even - // though only one `[media attached: media://...]` line is ever appended - // to the prompt (for the image offload). extractTrailingAttachmentMediaUris - // then read count=2 against one trailing URI, and - // mergePromptAttachmentImages placed the single offloaded image into the - // first "offloaded" slot — swapping it ahead of the inline image. + // though only one image offload existed. Structural facts and imageOrder + // must agree so hydration cannot swap it ahead of the inline image. const pdf = Buffer.from("%PDF-1.4\n").toString("base64"); const bigPng = Buffer.alloc(2_100_000); bigPng.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0); @@ -295,12 +293,16 @@ describe("parseMessageWithAttachments", () => { "image/png", ]); expect(parsed.imageOrder).toEqual(["inline", "offloaded"]); - // The offloaded-image URI is the sole trailing media:// line, matching - // imageOrder's single "offloaded" slot. + const pdfRef = expectDefined(parsed.offloadedRefs[0], "offloaded PDF ref"); + const imageRef = expectDefined(parsed.offloadedRefs[1], "offloaded image ref"); + expect(parsed.message).toBe( + `x\n[media attached: ${pdfRef.mediaRef}]\n[media attached: ${imageRef.mediaRef}]`, + ); + expect(parsed.messageWithoutOffloadedImageRefs).toBe(`x\n[media attached: ${pdfRef.mediaRef}]`); const trailingMediaLines = parsed.message .split("\n") .filter((line) => line.trim().startsWith("[media attached: media://inbound/")); - expect(trailingMediaLines).toHaveLength(1); + expect(trailingMediaLines).toHaveLength(2); }); it("rejects oversized images before offload", async () => { @@ -443,6 +445,7 @@ describe("parseMessageWithAttachments validation errors", () => { it("passes through unchanged on text-only session with no attachments", async () => { const { parsed } = await parseWithWarnings("hello", [], { supportsInlineImages: false }); expect(parsed.message).toBe("hello"); + expect(parsed.messageWithoutOffloadedImageRefs).toBe("hello"); expect(parsed.images).toHaveLength(0); expect(parsed.offloadedRefs).toHaveLength(0); expect(saveMediaBufferMock).not.toHaveBeenCalled(); @@ -468,7 +471,10 @@ describe("parseMessageWithAttachments validation errors", () => { expect(parsed.offloadedRefs).toHaveLength(1); expect(parsed.offloadedRefs[0]?.mimeType).toBe("application/pdf"); expect(parsed.offloadedRefs[0]?.label).toBe("brief.pdf"); - expect(parsed.message).toBe("read this"); + expect(parsed.message).toBe( + `read this\n[media attached: ${parsed.offloadedRefs[0]?.mediaRef}]`, + ); + expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message); } finally { await cleanupOffloadedRefs(parsed.offloadedRefs); } @@ -495,6 +501,7 @@ describe("parseMessageWithAttachments validation errors", () => { const offloaded = expectDefined(parsed.offloadedRefs[0], "offloaded image ref"); expect(offloaded.mimeType).toBe("image/png"); expect(parsed.message).toBe(`see this\n[media attached: ${offloaded.mediaRef}]`); + expect(parsed.messageWithoutOffloadedImageRefs).toBe("see this"); expect(parsed.media).toEqual([ { path: offloaded.path, @@ -530,6 +537,9 @@ describe("parseMessageWithAttachments validation errors", () => { expect(parsed.message).toContain( "[image attachment omitted: text-only attachment limit reached]", ); + expect(parsed.messageWithoutOffloadedImageRefs).toBe( + "see these\n[image attachment omitted: text-only attachment limit reached]", + ); expect(logs).toEqual([ "attachment dot-10.png: dropping image because text-only offload limit 10 was reached", ]); diff --git a/src/gateway/chat-attachments.ts b/src/gateway/chat-attachments.ts index 80106b1669a1..ad4a6895c3ec 100644 --- a/src/gateway/chat-attachments.ts +++ b/src/gateway/chat-attachments.ts @@ -36,6 +36,7 @@ export type OffloadedRef = { type ParsedMessageWithImages = { message: string; + messageWithoutOffloadedImageRefs: string; images: ChatImageContent[]; imageOrder: PromptImageOrderEntry[]; media: MediaFact[]; @@ -322,13 +323,21 @@ export async function parseMessageWithAttachments( const acceptNonImage = opts?.acceptNonImage !== false; if (!attachments || attachments.length === 0) { - return { message, images: [], imageOrder: [], media: [], offloadedRefs: [] }; + return { + message, + messageWithoutOffloadedImageRefs: message, + images: [], + imageOrder: [], + media: [], + offloadedRefs: [], + }; } const images: ChatImageContent[] = []; const imageOrder: PromptImageOrderEntry[] = []; const offloadedRefs: OffloadedRef[] = []; let updatedMessage = message; + let messageWithoutOffloadedImageRefs = message; let textOnlyImageOffloadCount = 0; const savedMediaIds: string[] = []; @@ -422,6 +431,8 @@ export async function parseMessageWithAttachments( `${TEXT_ONLY_OFFLOAD_LIMIT} was reached`, ); updatedMessage += "\n[image attachment omitted: text-only attachment limit reached]"; + messageWithoutOffloadedImageRefs += + "\n[image attachment omitted: text-only attachment limit reached]"; continue; } @@ -458,8 +469,10 @@ export async function parseMessageWithAttachments( savedMediaIds.push(savedMedia.id); const mediaRef = `media://inbound/${savedMedia.id}`; - if (isImage) { - updatedMessage += `\n[media attached: ${mediaRef}]`; + const mediaLine = `\n[media attached: ${mediaRef}]`; + updatedMessage += mediaLine; + if (!isImage) { + messageWithoutOffloadedImageRefs += mediaLine; } log?.info?.( shouldForceImageOffload && isImage @@ -491,6 +504,10 @@ export async function parseMessageWithAttachments( return { message: updatedMessage !== message ? updatedMessage.trimEnd() : message, + messageWithoutOffloadedImageRefs: + messageWithoutOffloadedImageRefs !== message + ? messageWithoutOffloadedImageRefs.trimEnd() + : message, images, imageOrder, media: offloadedRefs.map((ref) => ({ diff --git a/src/gateway/server-methods/chat-send-attachments.ts b/src/gateway/server-methods/chat-send-attachments.ts index cd5c324dcf68..fbf781feb925 100644 --- a/src/gateway/server-methods/chat-send-attachments.ts +++ b/src/gateway/server-methods/chat-send-attachments.ts @@ -57,23 +57,6 @@ function logAttachmentFailure( }); } -function stripTrailingOffloadedMediaMarkers(message: string, refs: OffloadedRef[]): string { - if (refs.length === 0) { - return message; - } - const removableRefs = new Set(refs.map((ref) => ref.mediaRef)); - const lines = message.split(/\r?\n/); - while (lines.length > 0) { - const last = lines[lines.length - 1]?.trim() ?? ""; - const match = /^\[media attached:\s*(media:\/\/inbound\/[^\]\s]+)\]$/.exec(last); - if (!match?.[1] || !removableRefs.delete(match[1])) { - break; - } - lines.pop(); - } - return lines.join("\n").trimEnd(); -} - function isPdfOffloadedRef(ref: OffloadedRef): boolean { const mime = ref.mimeType.trim().toLowerCase(); if (mime === "application/pdf" || mime.endsWith("+pdf")) { @@ -263,14 +246,11 @@ export async function prepareChatSendAttachments(params: { supportsImages, acceptNonImage: true, }); - parsedMessage = stripTrailingOffloadedMediaMarkers( - parsed.message, - routeImageOffloadsAsMediaPaths - ? parsed.offloadedRefs.filter((ref) => ref.mimeType.startsWith("image/")) - : [], - ); + parsedMessage = routeImageOffloadsAsMediaPaths + ? parsed.messageWithoutOffloadedImageRefs + : parsed.message; parsedImages = parsed.images; - imageOrder = routeImageOffloadsAsMediaPaths ? [] : parsed.imageOrder; + imageOrder = parsed.imageOrder; offloadedRefs = parsed.offloadedRefs; ({ paths: mediaPathOffloadPaths, diff --git a/src/gateway/server-methods/chat-send-user-turn.test.ts b/src/gateway/server-methods/chat-send-user-turn.test.ts index 39e5d2b7c2b0..7bc2c0e24b11 100644 --- a/src/gateway/server-methods/chat-send-user-turn.test.ts +++ b/src/gateway/server-methods/chat-send-user-turn.test.ts @@ -1,11 +1,21 @@ +import fs from "node:fs/promises"; +import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { GATEWAY_CLIENT_IDS, GATEWAY_CLIENT_MODES, type GatewayClientInfo, } from "../../../packages/gateway-protocol/src/client-info.js"; +import { createSolidPngBuffer } from "../../../test/helpers/image-fixtures.js"; +import { pruneProcessedHistoryImages } from "../../agents/embedded-agent-runner/run/history-image-prune.js"; +import { hydratePromptMediaMessages } from "../../agents/embedded-agent-runner/run/images.js"; +import type { AgentMessage } from "../../agents/runtime/index.js"; import type { MsgContext } from "../../auto-reply/templating.js"; -import type { UserTurnInput } from "../../sessions/user-turn-transcript.js"; +import { resolveStateDir } from "../../config/paths.js"; +import { + buildPersistedUserTurnMessage, + type UserTurnInput, +} from "../../sessions/user-turn-transcript.js"; import { applyChatSendManagedMediaFields, prepareChatSendUserTurn } from "./chat-send-user-turn.js"; function createUserTurnInputController() { @@ -206,8 +216,8 @@ describe("prepareChatSendUserTurn", () => { await expect(readInput()).resolves.toEqual(controller.baseInput); }); - it("carries retained image claim-check facts without changing the trailing prompt line", () => { - const { controller } = createUserTurnInputController(); + it("carries retained image claim-check facts without changing the trailing prompt line", async () => { + const { controller, readInput } = createUserTurnInputController(); const mediaRef = "media://inbound/image-1.png"; const prepared = prepareChatSendUserTurn({ request: { @@ -255,6 +265,166 @@ describe("prepareChatSendUserTurn", () => { contentType: "image/png", }, ]); + await expect(readInput()).resolves.toMatchObject({ + mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] }, + }); + }); + + it("persists and prunes the staged PDF claim-check alias as structured ownership", async () => { + const { controller, readInput } = createUserTurnInputController(); + const mediaRef = "media://inbound/report.pdf"; + prepareChatSendUserTurn({ + request: { + clientInfo: createClientInfo(), + normalizedAttachments: [{}], + suppressCommandInterpretation: false, + systemInputProvenance: undefined, + systemProvenanceReceipt: undefined, + }, + session: { + agentId: "main", + clientRunId: "run-1", + sessionKey: "agent:main:main", + }, + admission: { + originatingRoute: { originatingChannel: "webchat", explicitDeliverRoute: false }, + }, + attachments: createAttachments({ + offloadedRefs: [ + { + mediaRef, + id: "report.pdf", + path: "/media/inbound/report.pdf", + mimeType: "application/pdf", + label: "report.pdf", + sizeBytes: 10, + }, + ], + parsedMessage: `read this\n[media attached: ${mediaRef}]`, + }), + client: null, + logGateway: { warn: vi.fn() } as never, + userTurn: controller, + }); + + const input = await readInput(); + expect(input.media).toEqual([ + { + path: "/media/inbound/report.pdf", + url: mediaRef, + contentType: "application/pdf", + hydrationSuppressed: true, + }, + ]); + const persisted = buildPersistedUserTurnMessage({ + ...input, + text: `read this\n[media attached: ${mediaRef}]`, + }); + const history = [ + persisted, + { role: "assistant", content: "ack" }, + { role: "user", content: "more" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "more" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "more" }, + { role: "assistant", content: "ack" }, + ] as unknown as Parameters[0]; + const pruned = pruneProcessedHistoryImages(history); + const first = pruned?.[0] as unknown as Record | undefined; + expect(first?.content).toBe( + "read this\n[media reference removed - already processed by model]", + ); + expect((first?.["__openclaw"] as Record | undefined)?.media).toBeUndefined(); + }); + + it("hydrates and prunes a staged image claim-check alias as structured ownership", async () => { + const id = `gateway-image-${Date.now()}-${Math.random().toString(36).slice(2)}.png`; + const imagePath = path.join(resolveStateDir(), "media", "inbound", id); + const mediaRef = `media://inbound/${id}`; + const unownedRef = "media://inbound/unowned.png"; + const text = `inspect\n[media attached: ${mediaRef}]\n[media attached: ${unownedRef}]`; + await fs.mkdir(path.dirname(imagePath), { recursive: true }); + await fs.writeFile(imagePath, createSolidPngBuffer(2, 2, { r: 10, g: 20, b: 30 })); + + try { + const { controller, readInput } = createUserTurnInputController(); + prepareChatSendUserTurn({ + request: { + clientInfo: createClientInfo(), + normalizedAttachments: [{}], + suppressCommandInterpretation: false, + systemInputProvenance: undefined, + systemProvenanceReceipt: undefined, + }, + session: { + agentId: "main", + clientRunId: "run-1", + sessionKey: "agent:main:main", + }, + admission: { + originatingRoute: { originatingChannel: "webchat", explicitDeliverRoute: false }, + }, + attachments: createAttachments({ + imageOrder: ["offloaded"], + offloadedRefs: [ + { + mediaRef, + id, + path: imagePath, + mimeType: "image/png", + label: "image.png", + sizeBytes: 10, + }, + ], + parsedMessage: text, + }), + client: null, + logGateway: { warn: vi.fn() } as never, + userTurn: controller, + }); + + const input = await readInput(); + expect(input.media).toEqual([{ path: imagePath, url: mediaRef, contentType: "image/png" }]); + expect(input.media?.[0]).not.toHaveProperty("hydrationSuppressed"); + const persisted = buildPersistedUserTurnMessage({ ...input, text }); + expect( + ( + (persisted as unknown as Record)["__openclaw"] as { + media?: unknown; + } + ).media, + ).toEqual([{ path: imagePath, url: mediaRef, contentType: "image/png" }]); + + const hydrated = await hydratePromptMediaMessages([persisted as AgentMessage], { + workspaceDir: path.dirname(imagePath), + model: { input: ["text", "image"] }, + workspaceOnly: true, + }); + expect((hydrated[0] as unknown as { content?: unknown[] }).content).toEqual([ + { type: "text", text }, + expect.objectContaining({ type: "image", mimeType: "image/png" }), + ]); + + const history = [ + persisted, + { role: "assistant", content: "ack" }, + { role: "user", content: "more" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "more" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "more" }, + { role: "assistant", content: "ack" }, + ] as unknown as Parameters[0]; + const pruned = pruneProcessedHistoryImages(history); + const first = pruned?.[0] as unknown as Record | undefined; + expect(first?.content).toBe( + `inspect\n[media reference removed - already processed by model]\n[media attached: ${unownedRef}]`, + ); + expect((first?.["__openclaw"] as Record | undefined)?.media).toBeUndefined(); + } finally { + await fs.rm(imagePath, { force: true }); + } }); }); diff --git a/src/gateway/server-methods/chat-send-user-turn.ts b/src/gateway/server-methods/chat-send-user-turn.ts index 74d421e037f2..587370600e86 100644 --- a/src/gateway/server-methods/chat-send-user-turn.ts +++ b/src/gateway/server-methods/chat-send-user-turn.ts @@ -93,11 +93,26 @@ export function applyChatSendManagedMediaFields( } } -function buildChatSendUserTurnMedia(savedMedia: SavedMedia[]): NonNullable { - return savedMedia.map((entry) => ({ - path: entry.path, - contentType: entry.contentType, - })); +function buildChatSendUserTurnMedia( + savedMedia: SavedMedia[], + offloadedRefs: OffloadedRef[], +): NonNullable { + const offloadedRefsById = new Map(offloadedRefs.map((ref) => [ref.id, ref] as const)); + return savedMedia.map((entry) => { + const offloadedRef = offloadedRefsById.get(entry.id); + return { + path: entry.path, + ...(offloadedRef + ? { + // Every offload keeps its claim-check alias so persisted marker + // ownership survives; only non-images skip native image hydration. + url: offloadedRef.mediaRef, + ...(offloadedRef.mimeType.startsWith("image/") ? {} : { hydrationSuppressed: true }), + } + : {}), + contentType: entry.contentType, + }; + }); } function buildChatSendPromptMedia( @@ -251,10 +266,21 @@ export function prepareChatSendUserTurn(params: { ? getPersistedMediaForTranscript() : Promise.resolve([]); userTurn.setInputPromise( - preparedUserTurnMediaPromise.then(buildChatSendUserTurnMedia).then((media) => ({ - ...userTurn.baseInput, - ...(media.length > 0 ? { media } : {}), - })), + preparedUserTurnMediaPromise + .then((media) => buildChatSendUserTurnMedia(media, attachments.offloadedRefs)) + .then((media) => ({ + ...userTurn.baseInput, + ...(media.length > 0 ? { media } : {}), + ...(media.length > 0 && attachments.imageOrder.length > 0 + ? { + mediaImageLayout: { + // persistInboundImagesForTranscript emits image facts in this exact order, + // then appends non-images, so image slot ordinals are fact ordinals. + slots: attachments.imageOrder.map((kind, factIndex) => ({ kind, factIndex })), + }, + } + : {}), + })), ); const pluginBoundMediaFieldsPromise = attachments.explicitOriginTargetsPlugin && attachments.parsedImages.length > 0 diff --git a/src/gateway/server-methods/chat.directive-tags.test.ts b/src/gateway/server-methods/chat.directive-tags.test.ts index 9fc6fa79c431..51517e5e048c 100644 --- a/src/gateway/server-methods/chat.directive-tags.test.ts +++ b/src/gateway/server-methods/chat.directive-tags.test.ts @@ -5256,7 +5256,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () => | undefined; expect(mockState.lastDispatchImages).toBeUndefined(); expect(mockState.lastDispatchImageOrder).toBeUndefined(); - expect(mockState.lastDispatchCtx?.Body).toBe("summarize this"); + expect(mockState.lastDispatchCtx?.Body).toBe( + "summarize this\n[media attached: media://inbound/saved-media]", + ); expect(mockState.savedMediaCalls[0]?.contentType).toBe("application/pdf"); expect(mockState.savedMediaCalls[0]?.subdir).toBe("inbound"); expect(typeof mockState.savedMediaCalls[0]?.size).toBe("number"); @@ -5635,7 +5637,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(mockState.lastDispatchCtx?.Body).toBe("describe image"); }); expect(mockState.lastDispatchImages).toBeUndefined(); - expect(mockState.lastDispatchImageOrder).toBeUndefined(); + expect(mockState.lastDispatchImageOrder).toEqual(["offloaded"]); expect(mockState.lastDispatchCtx?.Body).toBe("describe image"); expect(mockState.lastDispatchCtx?.Body).not.toContain("media://"); expect(mockState.lastDispatchCtx?.MediaPath).toBe("/tmp/1.png"); @@ -5811,7 +5813,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(mockState.lastDispatchCtx?.Body).toBe("describe image"); }); expect(mockState.lastDispatchImages).toBeUndefined(); - expect(mockState.lastDispatchImageOrder).toBeUndefined(); + expect(mockState.lastDispatchImageOrder).toEqual(["offloaded"]); expect(mockState.lastDispatchCtx?.Body).toBe("describe image"); expect(mockState.lastDispatchCtx?.Body).not.toContain("media://"); expect(mockState.lastDispatchCtx?.MediaPath).toBe("/tmp/1.png"); @@ -5876,11 +5878,14 @@ describe("chat directive tag stripping for non-streaming final payloads", () => ); expect(mockState.lastDispatchCtx?.MediaTypes).toEqual(["application/pdf"]); expect(mockState.lastDispatchCtx?.MediaType).toBe("application/pdf"); - // Non-image offloads MUST NOT inject a media://URI into the prompt body — - // they ride through ctx.MediaPaths so buildInboundMediaNote prepends the - // real path, avoiding duplicate media markers. - expect(mockState.lastDispatchCtx?.Body).not.toContain("media://"); - expect(mockState.lastDispatchCtx?.BodyForAgent).not.toContain("media://"); + // Non-image offloads retain their claim-check line while the staged path + // also travels structurally for media tools and transcript persistence. + expect(mockState.lastDispatchCtx?.Body).toContain( + "[media attached: media://inbound/saved-media]", + ); + expect(mockState.lastDispatchCtx?.BodyForAgent).toContain( + "[media attached: media://inbound/saved-media]", + ); expect(mockState.lastDispatchImages).toBeUndefined(); // Marker replaces the implicit "relative-path no-op" coupling in // get-reply.ts with an explicit skip contract. @@ -5939,7 +5944,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () => ]); expect(mockState.lastDispatchCtx?.MediaTypes).toEqual(["application/zip"]); expect(mockState.lastDispatchImages).toBeUndefined(); - expect(mockState.lastDispatchCtx?.Body).not.toContain("media://"); + expect(mockState.lastDispatchCtx?.Body).toContain( + "[media attached: media://inbound/saved-media]", + ); expect(mockState.lastDispatchCtx?.MediaStaged).toBe(true); }); diff --git a/src/media/media-facts.ts b/src/media/media-facts.ts index 8eae408f87f8..b5ff3087deb5 100644 --- a/src/media/media-facts.ts +++ b/src/media/media-facts.ts @@ -1,6 +1,7 @@ import type { MediaKind } from "@openclaw/media-core/constants"; -import { kindFromMime } from "@openclaw/media-core/mime"; +import { kindFromMime, mimeTypeFromFilePath } from "@openclaw/media-core/mime"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import type { PromptImageOrderEntry } from "./prompt-image-order.js"; /** One ordered runtime attachment; array position is its alignment identity. */ export type MediaFact = { @@ -11,6 +12,10 @@ export type MediaFact = { transcribed?: boolean; messageId?: string; workspaceDir?: string; + // Declared field, not a symbol: suppression must survive every fact copy or + // reprojection boundary; described images otherwise rehydrate or count failed. + // Structured persistence may retain it; legacy Media* projections never emit it. + hydrationSuppressed?: boolean; }; export type MediaFactInput = { @@ -23,14 +28,66 @@ const RUNTIME_PROMPT_MEDIA_FACTS = Symbol.for("openclaw.runtimePromptMediaFacts" export function attachRuntimePromptMediaFacts( message: T, media: readonly MediaFact[], + imageOrder?: readonly PromptImageOrderEntry[], ): T { + const normalized = normalizeMediaFacts(media); + if (imageOrder?.length) { + Object.defineProperty(normalized, "imageOrder", { value: [...imageOrder] }); + } Object.defineProperty(message, RUNTIME_PROMPT_MEDIA_FACTS, { configurable: true, - value: normalizeMediaFacts(media), + value: normalized, }); return message; } +export function readRuntimePromptMediaFacts(message: object): MediaFact[] | undefined { + const media = (message as Record)[RUNTIME_PROMPT_MEDIA_FACTS]; + return Array.isArray(media) ? (media as MediaFact[]) : undefined; +} + +export function readRuntimePromptImageOrder(message: object): PromptImageOrderEntry[] | undefined { + const imageOrder = ( + readRuntimePromptMediaFacts(message) as + | (MediaFact[] & { imageOrder?: PromptImageOrderEntry[] }) + | undefined + )?.imageOrder; + return Array.isArray(imageOrder) ? (imageOrder as PromptImageOrderEntry[]) : undefined; +} + +/** Returns whether a fact can produce native image input. */ +export function isImageMediaFact(fact: MediaFactInput): boolean { + if (fact.kind && fact.kind !== "unknown") { + return fact.kind === "image" || fact.kind === "sticker"; + } + const contentType = normalizeOptionalString(fact.contentType); + const normalizedContentType = contentType?.split(";")[0]?.trim().toLowerCase(); + if ( + normalizedContentType && + normalizedContentType !== "application/octet-stream" && + normalizedContentType !== "binary/octet-stream" + ) { + const mimeKind = kindFromMime(normalizedContentType); + if (mimeKind) { + return mimeKind === "image"; + } + // Legacy channel-mode projections persist bare kinds as MediaType; honor + // them, and fall through to filename inference for other unknown strings. + if (normalizedContentType === "image" || normalizedContentType === "sticker") { + return true; + } + if ( + normalizedContentType === "audio" || + normalizedContentType === "video" || + normalizedContentType === "document" + ) { + return false; + } + } + const pathValue = normalizeOptionalString(fact.path) ?? normalizeOptionalString(fact.url); + return kindFromMime(mimeTypeFromFilePath(pathValue)) === "image"; +} + type MediaFactDefaults = { kind?: MediaKind; messageId?: string; @@ -61,7 +118,7 @@ function normalizeMediaFact( ): MediaFact { const workspaceDir = normalizeOptionalString(media.workspaceDir) ?? defaults.workspaceDir; const contentType = normalizeOptionalString(media.contentType); - return { + const normalized: MediaFact = { path: normalizeOptionalString(media.path), url: normalizeOptionalString(media.url), contentType, @@ -69,7 +126,9 @@ function normalizeMediaFact( transcribed: media.transcribed === true || defaults.transcribed?.(media, index) === true, messageId: normalizeOptionalString(media.messageId) ?? defaults.messageId, ...(workspaceDir ? { workspaceDir } : {}), + ...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}), }; + return normalized; } /** True when a consumer must use the already-staged legacy path projection. */ @@ -136,6 +195,7 @@ export function resolveMediaFacts(source: MediaFactSource): MediaFact[] { workspaceDir: normalizeOptionalString(fact?.workspaceDir) ?? normalizeOptionalString(source.MediaWorkspaceDir), + hydrationSuppressed: fact?.hydrationSuppressed, }, index, ); diff --git a/src/media/runtime-prompt-image-provenance.ts b/src/media/runtime-prompt-image-provenance.ts new file mode 100644 index 000000000000..5f011c229c8f --- /dev/null +++ b/src/media/runtime-prompt-image-provenance.ts @@ -0,0 +1,45 @@ +const RUNTIME_PROMPT_IMAGE_FACT_INDEXES = Symbol.for("openclaw.runtimePromptImageFactIndexes"); + +type RuntimePromptImageFactIndex = number | null; + +export function finalizeRuntimePromptImages( + entries: readonly { image: TImage; factIndex: RuntimePromptImageFactIndex }[], +): { images: TImage[]; imageFactIndexes: RuntimePromptImageFactIndex[] } { + const images = entries.map((entry) => entry.image); + const imageFactIndexes = entries.map((entry) => entry.factIndex); + attachRuntimePromptImageFactIndexes(images, imageFactIndexes); + return { images, imageFactIndexes }; +} + +/** Carries fact ownership on image blocks without changing provider-visible bytes. */ +function attachRuntimePromptImageFactIndexes( + images: readonly object[], + factIndexes: readonly RuntimePromptImageFactIndex[], +): void { + if (images.length !== factIndexes.length) { + return; + } + Object.defineProperty(images, RUNTIME_PROMPT_IMAGE_FACT_INDEXES, { + configurable: true, + value: [...factIndexes], + }); +} + +export function readRuntimePromptImageFactIndexes( + images: readonly object[] | null | undefined, +): RuntimePromptImageFactIndex[] | undefined { + if (!images?.length) { + return undefined; + } + const factIndexes = (images as unknown as Record)[ + RUNTIME_PROMPT_IMAGE_FACT_INDEXES + ]; + return Array.isArray(factIndexes) && + factIndexes.length === images.length && + factIndexes.every( + (entry) => + entry === null || (typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0), + ) + ? (factIndexes as RuntimePromptImageFactIndex[]) + : undefined; +} diff --git a/src/sessions/user-turn-transcript.media-normalize.ts b/src/sessions/user-turn-transcript.media-normalize.ts new file mode 100644 index 000000000000..bacfd28d7fbd --- /dev/null +++ b/src/sessions/user-turn-transcript.media-normalize.ts @@ -0,0 +1,97 @@ +import path from "node:path"; +import { mimeTypeFromFilePath } from "@openclaw/media-core/mime"; +import type { MediaFactInput } from "../media/media-facts.js"; +import type { PersistedUserTurnMediaInput } from "./user-turn-transcript.types.js"; + +const URL_LIKE_MEDIA_PATH_PATTERN = /^[a-z][a-z0-9+.-]*:/i; +const STRUCTURED_MEDIA_KINDS = new Set>([ + "image", + "audio", + "video", + "document", + "sticker", + "unknown", +]); + +function normalizeOptionalText(value: string | null | undefined): string | undefined { + const normalized = value?.trim(); + return normalized ? normalized : undefined; +} + +function mediaTypeForTranscript(media: PersistedUserTurnMediaInput, mediaPath?: string): string { + return ( + normalizeOptionalText(media.contentType) ?? + normalizeOptionalText(media.kind) ?? + mimeTypeFromFilePath(mediaPath) ?? + "application/octet-stream" + ); +} + +function normalizeStructuredMediaKind(value: string | null | undefined): MediaFactInput["kind"] { + const kind = normalizeOptionalText(value); + return kind && STRUCTURED_MEDIA_KINDS.has(kind as NonNullable) + ? (kind as NonNullable) + : undefined; +} + +export function resolveTranscriptMediaPath( + pathValue: string, + workspaceDir: string | undefined, +): string { + // Relative staged media paths are anchored to the media workspace; absolute + // paths and URL-like refs are already stable transcript references. + if (!workspaceDir || path.isAbsolute(pathValue) || URL_LIKE_MEDIA_PATH_PATTERN.test(pathValue)) { + return pathValue; + } + return path.join(workspaceDir, pathValue); +} + +export function normalizeMediaEntryForTranscript( + media: PersistedUserTurnMediaInput, +): MediaFactInput { + const rawPath = normalizeOptionalText(media.path) ?? normalizeOptionalText(media.url); + if (!rawPath) { + return media.hydrationSuppressed === true + ? { + contentType: normalizeOptionalText(media.contentType), + hydrationSuppressed: true, + } + : {}; + } + return { + path: resolveTranscriptMediaPath(rawPath, normalizeOptionalText(media.workspaceDir)), + contentType: mediaTypeForTranscript(media, rawPath), + ...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}), + }; +} + +export function normalizeStructuredMediaEntryForTranscript( + media: PersistedUserTurnMediaInput, +): MediaFactInput { + const mediaPath = normalizeOptionalText(media.path); + const mediaUrl = normalizeOptionalText(media.url); + return { + path: mediaPath, + url: mediaUrl, + contentType: + normalizeOptionalText(media.contentType) ?? mimeTypeFromFilePath(mediaPath ?? mediaUrl), + kind: normalizeStructuredMediaKind(media.kind), + workspaceDir: normalizeOptionalText(media.workspaceDir), + ...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}), + }; +} + +export function shouldPersistStructuredMediaEntries( + media: readonly PersistedUserTurnMediaInput[] | null | undefined, +): boolean { + return (media ?? []).some((entry) => { + const legacy = normalizeMediaEntryForTranscript(entry); + const structured = normalizeStructuredMediaEntryForTranscript(entry); + const structuredIdentity = structured.path ?? structured.url; + return ( + structured.hydrationSuppressed === true || + structuredIdentity !== legacy.path || + Boolean(structured.url && structured.url !== legacy.path) + ); + }); +} diff --git a/src/sessions/user-turn-transcript.ts b/src/sessions/user-turn-transcript.ts index d96f28b4d7b9..e2f41d6df5c6 100644 --- a/src/sessions/user-turn-transcript.ts +++ b/src/sessions/user-turn-transcript.ts @@ -1,18 +1,18 @@ // User turn transcript helpers extract user-turn text from session transcripts. -import path from "node:path"; import { mimeTypeFromFilePath } from "@openclaw/media-core/mime"; import type { AgentMessage } from "../../packages/agent-core/src/types.js"; import { persistSessionTranscriptTurn, type SessionTranscriptTurnPersistOptions, } from "../config/sessions/session-accessor.js"; -import { - projectMediaFacts, - resolveMediaFacts, - type MediaFact, - type MediaFactInput, -} from "../media/media-facts.js"; +import { projectMediaFacts, resolveMediaFacts, type MediaFact } from "../media/media-facts.js"; import { applyInputProvenanceToUserMessage, normalizeInputProvenance } from "./input-provenance.js"; +import { + normalizeMediaEntryForTranscript, + normalizeStructuredMediaEntryForTranscript, + resolveTranscriptMediaPath, + shouldPersistStructuredMediaEntries, +} from "./user-turn-transcript.media-normalize.js"; import type { CreateUserTurnTranscriptRecorderParams, PersistUserTurnTranscriptParams, @@ -72,37 +72,6 @@ export function resolvePersistedUserTurnText(value: string | null | undefined): return normalized; } -function mediaTypeForTranscript(media: PersistedUserTurnMediaInput, mediaPath?: string): string { - return ( - normalizeOptionalText(media.contentType) ?? - normalizeOptionalText(media.kind) ?? - mimeTypeFromFilePath(mediaPath) ?? - "application/octet-stream" - ); -} - -function normalizeMediaEntryForTranscript(media: PersistedUserTurnMediaInput): MediaFactInput { - const rawPath = normalizeOptionalText(media.path) ?? normalizeOptionalText(media.url); - if (!rawPath) { - return {}; - } - return { - path: resolveTranscriptMediaPath(rawPath, normalizeOptionalText(media.workspaceDir)), - contentType: mediaTypeForTranscript(media, rawPath), - }; -} - -const URL_LIKE_MEDIA_PATH_PATTERN = /^[a-z][a-z0-9+.-]*:/i; - -function resolveTranscriptMediaPath(pathValue: string, workspaceDir: string | undefined): string { - // Relative staged media paths are anchored to the media workspace; absolute - // paths and URL-like refs are already stable transcript references. - if (!workspaceDir || path.isAbsolute(pathValue) || URL_LIKE_MEDIA_PATH_PATTERN.test(pathValue)) { - return pathValue; - } - return path.join(workspaceDir, pathValue); -} - function resolveTranscriptMediaType(params: { explicitType: string | undefined; mediaPath: string | undefined; @@ -211,6 +180,7 @@ function readOpenClawMessageMeta(message: AgentMessage): Record export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedUserTurnMessage { const mediaFields = buildPersistedUserTurnMediaFields(params.media); + const normalizedMedia = (params.media ?? []).map(normalizeStructuredMediaEntryForTranscript); const text = normalizeTranscriptText(params.text); // Storage is BARE (no timestamp prefix). The per-message timestamp is added // at the single LLM-boundary stamping site (normalizeMessagesForLlmBoundary), @@ -223,6 +193,19 @@ export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedU ...(params.senderIsOwner === undefined ? {} : { senderIsOwner: params.senderIsOwner }), ...senderMeta, ...(params.transport ? { transport: params.transport } : {}), + ...(shouldPersistStructuredMediaEntries(params.media) ? { media: normalizedMedia } : {}), + ...(params.mediaImageLayout + ? { + mediaImageLayout: { + slots: params.mediaImageLayout.slots.map((slot) => ({ ...slot })), + ...(params.mediaImageLayout.suppressedFactIndexes?.length + ? { + suppressedFactIndexes: [...params.mediaImageLayout.suppressedFactIndexes], + } + : {}), + }, + } + : {}), }; const message = { role: "user", diff --git a/src/sessions/user-turn-transcript.types.ts b/src/sessions/user-turn-transcript.types.ts index a42caa218d1b..d5f191a85edd 100644 --- a/src/sessions/user-turn-transcript.types.ts +++ b/src/sessions/user-turn-transcript.types.ts @@ -14,7 +14,10 @@ type UserTurnSessionEntry = { threadId?: string | number; } & Record; -export type PersistedUserTurnMediaInput = Pick & { +export type PersistedUserTurnMediaInput = Pick< + MediaFactInput, + "contentType" | "hydrationSuppressed" | "path" | "url" +> & { kind?: string | null; workspaceDir?: string | null; }; @@ -24,6 +27,14 @@ export type PersistedUserTurnMessage = Extract; export type UserTurnInput = { text?: string | null; media?: readonly PersistedUserTurnMediaInput[] | null; + /** Restart-safe native image placement; model-visible prompt bytes remain separate. */ + mediaImageLayout?: { + slots: readonly { + kind: "inline" | "offloaded"; + factIndex?: number; + }[]; + suppressedFactIndexes?: readonly number[]; + } | null; timestamp?: number; idempotencyKey?: string; senderIsOwner?: boolean;