refactor(media): hydrate and prune images from structured facts, delete marker-text parsing (#112913)

* refactor(media): hydrate and prune images from structured facts, delete marker-text parsing

Embedded, CLI, host-context replay, and plugin-harness image hydration now
consume the persisted/runtime MediaFact carrier instead of parsing
OpenClaw-authored [media attached:]/[Image: source:] marker text. The
prompt-text attachment parser family (splitPromptAndAttachmentRefs, leading/
trailing marker extractors, ref-count consumption, text-based prune scrubbing)
is deleted; explicit user-authored paths and file URLs still hydrate.

- Described-image suppression is a declared MediaFact.hydrationSuppressed
  field carried across copy/reprojection boundaries; the runtime symbol
  plumbing is removed.
- Persisted facts plus mediaImageLayout are authoritative at every hydration
  entry point; image blocks reconcile by structural fact index, never
  byte/MIME equality.
- Offloaded attachments (image and non-image) persist their media://inbound
  claim-check alias so ownership-aware history pruning can scrub markers
  after persistence; factless pre-MediaFact rows keep narrow legacy redaction.
- History pruning removes facts, layout, and block provenance structurally
  before replay.

Prompt bytes are golden-equal throughout; goldens from the fact-carrier PR
are untouched. 1,303 tests across 44 files; delegated check:changed green
(run 29980534186, pinned to the declared merge-base).

* fix(media): fail missing inline image slots

* test(agents): carry offloaded CLI retry image as a structured fact

The retry-image reliability test modeled an offloaded attachment with only
its [media attached:] marker text; offloaded attachments are carried as
structured media facts and marker text is presentation-only, never parsed
for hydration.
This commit is contained in:
Peter Steinberger
2026-07-23 03:11:16 -04:00
committed by GitHub
parent b9a28cde42
commit f6818ae5ea
48 changed files with 4225 additions and 941 deletions
+80 -61
View File
@@ -3,11 +3,11 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "@openclaw/ai/internal/shared";
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
import { expectDefined } from "@openclaw/normalization-core";
import type { ImageContent } from "openclaw/plugin-sdk/llm";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { createSolidPngBuffer } from "../../test/helpers/image-fixtures.js";
import { buildInboundMediaNoteProjection } from "../auto-reply/media-note.js";
import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js";
import { escapeRegExp } from "../shared/regexp.js";
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
@@ -28,7 +28,7 @@ describe("prepareCliPromptImagePayload prompt references", () => {
});
it("returns empty results when the prompt has no image refs", async () => {
const loadImageFromRefSpy = vi.spyOn(promptImageUtils, "loadImageFromRef");
const detectAndLoadPromptImagesSpy = vi.spyOn(promptImageUtils, "detectAndLoadPromptImages");
const sanitizeImageBlocksSpy = vi.spyOn(toolImages, "sanitizeImageBlocks");
await expect(
@@ -39,12 +39,12 @@ describe("prepareCliPromptImagePayload prompt references", () => {
}),
).resolves.toStrictEqual({ prompt: "just text" });
expect(loadImageFromRefSpy).not.toHaveBeenCalled();
expect(detectAndLoadPromptImagesSpy).not.toHaveBeenCalled();
expect(sanitizeImageBlocksSpy).not.toHaveBeenCalled();
});
it("does not reload OpenClaw CLI image cache paths from prior prompt text", async () => {
const loadImageFromRefSpy = vi.spyOn(promptImageUtils, "loadImageFromRef");
const detectAndLoadPromptImagesSpy = vi.spyOn(promptImageUtils, "detectAndLoadPromptImages");
const sanitizeImageBlocksSpy = vi.spyOn(toolImages, "sanitizeImageBlocks");
await expect(
@@ -59,99 +59,115 @@ describe("prepareCliPromptImagePayload prompt references", () => {
});
// Cached image paths are generated output, not fresh user references.
expect(loadImageFromRefSpy).not.toHaveBeenCalled();
expect(detectAndLoadPromptImagesSpy).not.toHaveBeenCalled();
expect(sanitizeImageBlocksSpy).not.toHaveBeenCalled();
});
it("passes the max-byte guardrail through load and sanitize", async () => {
const loadedImage: ImageContent = {
type: "image",
data: "c29tZS1pbWFnZQ==",
mimeType: "image/png",
};
const sanitizedImage: ImageContent = {
type: "image",
data: "c2FuaXRpemVkLWltYWdl",
mimeType: "image/jpeg",
};
it("hydrates explicit prompt refs through the shared image loader", async () => {
const workspaceDir = await fs.mkdtemp(
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-ref-image-"),
);
const loadImageFromRefSpy = vi
.spyOn(promptImageUtils, "loadImageFromRef")
.mockResolvedValueOnce(loadedImage);
const sanitizeImageBlocksSpy = vi
.spyOn(toolImages, "sanitizeImageBlocks")
.mockResolvedValueOnce({ images: [sanitizedImage], dropped: 0 });
const imagePath = path.join(workspaceDir, "photo.png");
const image = createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 });
await fs.writeFile(imagePath, image);
try {
const result = await prepareCliPromptImagePayload({
backend: { command: "gemini", imagePathScope: "workspace" },
prompt: "Look at /tmp/photo.png",
prompt: `Look at ${imagePath}`,
workspaceDir,
});
const [ref, loadedWorkspaceDir, options] = loadImageFromRefSpy.mock.calls[0] ?? [];
expect(ref?.resolved).toBe("/tmp/photo.png");
expect(ref?.type).toBe("path");
expect(loadedWorkspaceDir).toBe(workspaceDir);
expect(options).toEqual({
maxBytes: MAX_IMAGE_BYTES,
workspaceOnly: undefined,
sandbox: undefined,
});
expect(sanitizeImageBlocksSpy).toHaveBeenCalledWith([loadedImage], "prompt:images", {
maxBytes: MAX_IMAGE_BYTES,
});
expect(result.imagePaths).toHaveLength(1);
await expect(
fs.readFile(expectDefined(result.imagePaths?.[0], "image path")),
).resolves.toEqual(Buffer.from(sanitizedImage.data, "base64"));
).resolves.toEqual(image);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("dedupes repeated refs and skips failed loads before sanitizing", async () => {
const loadedImage: ImageContent = {
type: "image",
data: "b25lLWltYWdl",
mimeType: "image/png",
};
const loadImageFromRefSpy = vi
.spyOn(promptImageUtils, "loadImageFromRef")
.mockResolvedValueOnce(loadedImage)
.mockResolvedValueOnce(null);
const sanitizeImageBlocksSpy = vi
.spyOn(toolImages, "sanitizeImageBlocks")
.mockResolvedValueOnce({ images: [loadedImage], dropped: 0 });
const workspaceDir = await fs.mkdtemp(
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-ref-dedupe-"),
);
const imagePath = path.join(workspaceDir, "a.png");
await fs.writeFile(imagePath, createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 }));
try {
const result = await prepareCliPromptImagePayload({
backend: { command: "gemini", imagePathScope: "workspace" },
prompt: "Compare /tmp/a.png with /tmp/a.png and /tmp/b.png",
prompt: `Compare ${imagePath} with ${imagePath} and ${path.join(workspaceDir, "missing.png")}`,
workspaceDir,
});
expect(loadImageFromRefSpy).toHaveBeenCalledTimes(2);
expect(
loadImageFromRefSpy.mock.calls.map(
(call) => (call[0] as { resolved?: string } | undefined)?.resolved,
),
).toEqual(["/tmp/a.png", "/tmp/b.png"]);
expect(sanitizeImageBlocksSpy).toHaveBeenCalledWith([loadedImage], "prompt:images", {
maxBytes: MAX_IMAGE_BYTES,
});
expect(result.imagePaths).toHaveLength(1);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("surfaces structured image hydration failures", async () => {
const workspaceDir = await fs.mkdtemp(
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-structured-failure-"),
);
try {
await expect(
prepareCliPromptImagePayload({
backend: { command: "codex" },
prompt: "describe the attachment",
workspaceDir,
media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }],
}),
).rejects.toThrow("failed to hydrate 1 structured image attachment");
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("surfaces inline sanitization failure when a preceding image fact is suppressed", async () => {
await expect(
prepareCliPromptImagePayload({
backend: { command: "codex" },
prompt: "already described",
workspaceDir: "/tmp",
images: [{ type: "image", data: "%%%", mimeType: "image/png" }],
imageOrder: ["inline"],
media: [
{
path: "/tmp/described-missing.png",
contentType: "image/png",
hydrationSuppressed: true,
},
{ path: "/tmp/inline.png", contentType: "image/png" },
],
}),
).rejects.toThrow("failed to hydrate 1 structured image attachment");
});
it("accepts an intentionally non-hydrating remote-only image fact", async () => {
const media = buildInboundMediaNoteProjection({
MediaPaths: [""],
MediaUrls: ["https://example.com/described.png"],
MediaTypes: ["image/png"],
MediaUnderstanding: [
{
kind: "image.description",
attachmentIndex: 0,
text: "already described",
provider: "test",
},
],
}).media;
await expect(
prepareCliPromptImagePayload({
backend: { command: "codex" },
prompt: "already described",
workspaceDir: "/tmp",
media,
}),
).resolves.toEqual({ prompt: "already described" });
});
});
describe("buildCliArgs", () => {
@@ -366,6 +382,7 @@ describe("writeCliImages", () => {
input: "arg",
},
prompt: `[media attached: ${sourceImage} (image/png)]\n\n<media:image>`,
media: [{ path: sourceImage, contentType: "image/png" }],
workspaceDir: tempDir,
});
const argv = buildCliArgs({
@@ -411,6 +428,7 @@ describe("writeCliImages", () => {
input: "stdin",
},
prompt,
media: [{ path: sourceImage, contentType: "image/png" }],
workspaceDir: tempDir,
});
const promptWithImages = prepared.prompt;
@@ -560,6 +578,7 @@ describe("writeCliImages", () => {
},
],
imageOrder: ["offloaded", "inline"],
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
});
expect(prepared.imagePaths).toHaveLength(2);
@@ -689,6 +689,9 @@ describe("runCliAgent reliability", () => {
},
],
imageOrder: ["offloaded", "inline"],
// Offloaded attachments are carried as structured facts; the trailing
// marker text is presentation only and is never parsed for hydration.
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
};
const result = await runPreparedCliAgent(context);
+41
View File
@@ -593,9 +593,50 @@ describe("runCliAgent spawn path", () => {
await expect(executePreparedCliRun(context)).rejects.toThrow(
"paired-node Claude CLI sessions do not support attachments or images",
);
context.params.imagePrompt = undefined;
context.params.media = [{ path: "/tmp/hydratable.png", kind: "image" }];
await expect(executePreparedCliRun(context)).rejects.toThrow(
"paired-node Claude CLI sessions do not support attachments or images",
);
expect(invokeNode).not.toHaveBeenCalled();
});
it("allows non-hydratable image facts on a text-only node turn", async () => {
const invokeNode = vi.fn(async (params: Parameters<typeof invokeNodeClaudeCliRun>[0]) => {
params.onProgress(
[
JSON.stringify({ type: "system", subtype: "init", session_id: "node-text-only" }),
JSON.stringify({ type: "result", session_id: "node-text-only", result: "ok" }),
"",
].join("\n"),
);
return {
ok: true,
payloadJSON: JSON.stringify({ exitCode: 0, stderrTail: "", truncated: false }),
};
});
setCliRunnerExecuteTestDeps({ invokeNodeClaudeCliRun: invokeNode });
const context = buildPreparedCliRunContext({
provider: "claude-cli",
model: "claude-opus-4-8",
runId: "run-node-text-only-media-facts",
prompt: "already described",
sessionEntry: {
sessionId: "openclaw-session",
updatedAt: 1,
execHost: "node",
execNode: "node-a",
},
});
context.params.media = [
{ kind: "image" },
{ kind: "image", url: "https://example.test/described.png" },
];
await expect(executePreparedCliRun(context)).resolves.toMatchObject({ text: "ok" });
expect(invokeNode).toHaveBeenCalledOnce();
});
it("does not inject hardcoded 'Tools are disabled' text into CLI arguments", async () => {
supervisorSpawnMock.mockResolvedValueOnce(
createManagedRun({
+11 -1
View File
@@ -68,6 +68,10 @@ import type {
MessagingToolSend,
MessagingToolSourceReplyPayload,
} from "../embedded-agent-messaging.types.js";
import {
detectImageReferences,
hasHydratableMediaImages,
} from "../embedded-agent-runner/run/images.js";
import {
extractMessagingToolSendResult,
extractMessagingToolSourceReplyPayload,
@@ -458,7 +462,12 @@ export async function executePreparedCliRun(
}),
context.backendResolved.textTransforms?.input,
);
if (nodePlacement && ((params.images?.length ?? 0) > 0 || Boolean(params.imagePrompt?.trim()))) {
if (
nodePlacement &&
((params.images?.length ?? 0) > 0 ||
hasHydratableMediaImages(params.media) ||
(params.imagePrompt ? detectImageReferences(params.imagePrompt).length > 0 : false))
) {
throw new Error("paired-node Claude CLI sessions do not support attachments or images");
}
const {
@@ -474,6 +483,7 @@ export async function executePreparedCliRun(
workspaceDir: context.workspaceDir,
images: params.images,
imageOrder: params.imageOrder,
media: params.media,
});
prompt = promptWithImages;
+24 -56
View File
@@ -23,6 +23,7 @@ import { privateFileStore } from "../../infra/private-file-store.js";
import { tempWorkspace } from "../../infra/private-temp-workspace.js";
import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js";
import type { ImageContent } from "../../llm/types.js";
import type { MediaFact } from "../../media/media-facts.js";
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
import { KeyedAsyncQueue } from "../../plugin-sdk/keyed-async-queue.js";
import type { CliBackendConfig } from "../../plugins/cli-backend.types.js";
@@ -32,16 +33,13 @@ import type { EmbeddedContextFile } from "../embedded-agent-helpers.js";
import {
detectAndLoadPromptImages,
detectImageReferences,
loadImageFromRef,
} from "../embedded-agent-runner/run/images.js";
import { resolveDefaultModelForAgent } from "../model-selection.js";
import type { AgentTool } from "../runtime/index.js";
import type { SandboxFsBridge } from "../sandbox/fs-bridge.js";
import { detectRuntimeShell } from "../shell-utils.js";
import { buildConfiguredAgentSystemPrompt } from "../system-prompt-config.js";
import { buildSystemPromptParams } from "../system-prompt-params.js";
import type { SilentReplyPromptMode } from "../system-prompt.types.js";
import { sanitizeImageBlocks } from "../tool-images.js";
import { cliBackendLog } from "./log.js";
import { formatTomlConfigOverride } from "./toml-inline.js";
/** Re-export CLI reliability helpers used by older runner call sites. */
@@ -362,44 +360,6 @@ function appendImagePathsToPrompt(prompt: string, paths: string[], prefix = ""):
return `${trimmed}${separator}${paths.map((entry) => `${prefix}${entry}`).join("\n")}`;
}
/** Loads and sanitizes image references found in prompt text. */
async function loadPromptRefImages(params: {
prompt: string;
workspaceDir: string;
maxBytes?: number;
workspaceOnly?: boolean;
sandbox?: { root: string; bridge: SandboxFsBridge };
}): Promise<ImageContent[]> {
const refs = detectImageReferences(params.prompt);
if (refs.length === 0) {
return [];
}
const maxBytes = params.maxBytes ?? MAX_IMAGE_BYTES;
const seen = new Set<string>();
const images: ImageContent[] = [];
for (const ref of refs) {
const key = `${ref.type}:${ref.resolved}`;
if (seen.has(key)) {
continue;
}
seen.add(key);
const image = await loadImageFromRef(ref, params.workspaceDir, {
maxBytes,
workspaceOnly: params.workspaceOnly,
sandbox: params.sandbox,
});
if (image) {
images.push(image);
}
}
const { images: sanitizedImages } = await sanitizeImageBlocks(images, "prompt:images", {
maxBytes,
});
return sanitizedImages;
}
/** Writes CLI image payloads to private paths and returns their file paths. */
async function writeCliImages(params: {
backend: CliBackendConfig;
@@ -459,27 +419,35 @@ export async function prepareCliPromptImagePayload(params: {
workspaceDir: string;
images?: ImageContent[];
imageOrder?: PromptImageOrderEntry[];
media?: MediaFact[];
}): Promise<{
prompt: string;
imagePaths?: string[];
cleanupImages?: () => Promise<void>;
}> {
let prompt = params.prompt;
const resolvedImages =
params.imagePrompt !== undefined
? (
await detectAndLoadPromptImages({
prompt: params.imagePrompt,
workspaceDir: params.workspaceDir,
model: { input: ["text", "image"] },
existingImages: params.images,
imageOrder: params.imageOrder,
maxBytes: MAX_IMAGE_BYTES,
})
).images
: params.images && params.images.length > 0
? params.images
: await loadPromptRefImages({ prompt, workspaceDir: params.workspaceDir });
const imagePrompt = params.imagePrompt ?? prompt;
const needsHydration =
params.imagePrompt !== undefined ||
Boolean(params.media?.length) ||
(!params.images?.length && detectImageReferences(imagePrompt).length > 0);
const imageResult = needsHydration
? await detectAndLoadPromptImages({
prompt: imagePrompt,
media: params.media,
workspaceDir: params.workspaceDir,
model: { input: ["text", "image"] },
existingImages: params.images,
imageOrder: params.imageOrder,
maxBytes: MAX_IMAGE_BYTES,
})
: undefined;
if (imageResult?.failedMediaCount) {
throw new Error(
`failed to hydrate ${imageResult.failedMediaCount} structured image attachment(s) for CLI input`,
);
}
const resolvedImages = imageResult?.images ?? params.images ?? [];
if (resolvedImages.length === 0) {
return { prompt };
}
@@ -193,6 +193,7 @@ async function runEmbeddedAgentViaCliBackend(
agentDir: params.agentDir,
config: params.config,
prompt: params.prompt,
imagePrompt: params.prompt,
media: params.media,
provider: dispatch.provider,
model: params.model,
@@ -1,9 +1,12 @@
/** Installs attempt-local context engine, tool-result, image, and frame guards. */
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../../context-engine/host-compat.js";
import { buildContextEngineRuntimeSettings } from "../../../context-engine/runtime-settings.js";
import type { ContextEngine } from "../../../context-engine/types.js";
import { isHeartbeatLifecycleRunKind } from "../../bootstrap-mode.js";
import { DEFAULT_CONTEXT_TOKENS } from "../../defaults.js";
import { resolveImageSanitizationLimits } from "../../image-sanitization.js";
import type { SandboxContext } from "../../sandbox/types.js";
import type { guardSessionManager } from "../../session-tool-result-guard-wrapper.js";
import type { AgentSession } from "../../sessions/index.js";
import { invalidateComputerFrameIfMissing } from "../../tools/computer-tool.js";
@@ -29,6 +32,7 @@ export function installEmbeddedAttemptContextGuards(input: {
attempt: EmbeddedRunAttemptParams;
computerContextEpoch: { value: number };
effectiveCwd: string;
effectiveFsWorkspaceOnly: boolean;
effectiveWorkspace: string;
getPrePromptMessageCount: () => number;
getPromptCache: () => EmbeddedRunAttemptResult["promptCache"];
@@ -39,6 +43,7 @@ export function installEmbeddedAttemptContextGuards(input: {
sessionAgentId: string;
sessionManager: ReturnType<typeof guardSessionManager>;
settingsManager: AgentSession["settingsManager"];
sandbox?: SandboxContext | null;
}): {
getAfterTurnCheckpoint: () => number | null;
remove: () => void;
@@ -152,6 +157,17 @@ export function installEmbeddedAttemptContextGuards(input: {
const removeHistoryImagePruneContextTransform = installHistoryImagePruneContextTransform(
activeSession.agent,
{
workspaceDir: input.effectiveWorkspace,
model: attempt.model,
maxBytes: MAX_IMAGE_BYTES,
maxDimensionPx: resolveImageSanitizationLimits(attempt.config).maxDimensionPx,
workspaceOnly: input.effectiveFsWorkspaceOnly,
sandbox:
input.sandbox?.enabled && input.sandbox.fsBridge
? { root: input.sandbox.workspaceDir, bridge: input.sandbox.fsBridge }
: undefined,
},
);
const previousComputerFrameTransform = activeSession.agent.transformContext;
activeSession.agent.transformContext = async (messages, signal) => {
@@ -65,7 +65,9 @@ describe("dispatchEmbeddedAttemptPrompt", () => {
vi.clearAllMocks();
hoisted.prepareEmbeddedAttemptPromptExecution.mockResolvedValue({
images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
imageFactIndexes: [null],
detectedRefs: [],
failedMediaCount: 0,
loadedCount: 1,
skippedCount: 0,
});
@@ -82,7 +84,9 @@ describe("dispatchEmbeddedAttemptPrompt", () => {
order.push("images");
return {
images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
imageFactIndexes: [null],
detectedRefs: [],
failedMediaCount: 0,
loadedCount: 1,
skippedCount: 0,
};
@@ -12,7 +12,10 @@ const hoisted = vi.hoisted(() => ({
withSessionWriteLock: vi.fn(async (operation: () => unknown) => await operation()),
}));
vi.mock("@openclaw/media-core/constants", () => ({ MAX_IMAGE_BYTES: 1_234 }));
vi.mock("@openclaw/media-core/constants", () => ({
MAX_IMAGE_BYTES: 1_234,
mediaKindFromMime: (mime: string) => (mime.startsWith("image/") ? "image" : "unknown"),
}));
vi.mock("../../image-sanitization.js", () => ({
resolveImageSanitizationLimits: hoisted.resolveImageSanitizationLimits,
}));
@@ -70,7 +73,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
hoisted.resolveImageSanitizationLimits.mockReturnValue({ maxDimensionPx: 2048 });
hoisted.detectAndLoadPromptImages.mockResolvedValue({
images: [{ type: "image", data: "loaded", mimeType: "image/png" }],
imageFactIndexes: [null],
detectedRefs: [],
failedMediaCount: 0,
loadedCount: 1,
skippedCount: 0,
});
@@ -87,7 +92,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
expect(second).toEqual({
images: [],
imageFactIndexes: [],
detectedRefs: [],
failedMediaCount: 0,
loadedCount: 0,
skippedCount: 0,
});
@@ -136,7 +143,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
});
expect(result).toEqual({
images: [{ type: "image", data: "loaded", mimeType: "image/png" }],
imageFactIndexes: [null],
detectedRefs: [],
failedMediaCount: 0,
loadedCount: 1,
skippedCount: 0,
});
@@ -152,4 +161,81 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
expect.objectContaining({ sandbox: undefined }),
);
});
it("reports failed hydration without consuming the embedded attempt fact", async () => {
const base = createInput();
const media = [{ path: "/tmp/missing.png", contentType: "image/png" }];
const input = createInput({ attempt: { ...base.attempt, media } });
hoisted.detectAndLoadPromptImages.mockResolvedValueOnce({
images: [],
imageFactIndexes: [],
detectedRefs: [],
failedMediaCount: 1,
loadedCount: 0,
skippedCount: 1,
});
const result = await prepareEmbeddedAttemptPromptExecution(input);
expect(result.failedMediaCount).toBe(1);
expect(input.attempt.media).toBe(media);
});
it("uses persisted facts and layout as the current-turn provenance authority", async () => {
const base = createInput();
const persistedMessage = {
role: "user" as const,
content: "compare",
MediaPaths: ["/tmp/inline.png", "/tmp/offloaded.png"],
MediaTypes: ["image/png", "image/png"],
__openclaw: {
media: [
{
path: "/tmp/inline.png",
contentType: "image/png",
hydrationSuppressed: true,
},
{ path: "/tmp/offloaded.png", contentType: "image/png" },
],
mediaImageLayout: {
slots: [
{ kind: "inline", factIndex: 0 },
{ kind: "offloaded", factIndex: 1 },
],
},
},
};
const input = createInput({
attempt: {
...base.attempt,
media: [{ path: "/tmp/offloaded.png", contentType: "image/png" }],
userTurnTranscriptRecorder: {
message: persistedMessage,
resolveMessage: vi.fn(async () => persistedMessage),
} as unknown as NonNullable<PromptExecutionInput["attempt"]["userTurnTranscriptRecorder"]>,
},
});
await prepareEmbeddedAttemptPromptExecution(input);
expect(hoisted.detectAndLoadPromptImages).toHaveBeenCalledWith(
expect.objectContaining({
media: [
expect.objectContaining({
path: "/tmp/inline.png",
kind: "image",
hydrationSuppressed: true,
}),
expect.objectContaining({ path: "/tmp/offloaded.png", kind: "image" }),
],
mediaImageLayout: {
slots: [
{ kind: "inline", factIndex: 0 },
{ kind: "offloaded", factIndex: 1 },
],
suppressedFactIndexes: [],
},
}),
);
});
});
@@ -1,6 +1,7 @@
/** Prepares prompt-lock ownership and prompt-local images for submission. */
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
import type { OwnedSessionTranscriptCacheSnapshot } from "../../../config/sessions/transcript-write-context.js";
import { resolveMediaFacts } from "../../../media/media-facts.js";
import { resolveImageSanitizationLimits } from "../../image-sanitization.js";
import type { SandboxContext } from "../../sandbox/types.js";
import type { AgentSession } from "../../sessions/index.js";
@@ -9,18 +10,31 @@ import {
installPromptSubmissionLockRelease,
} from "./attempt.session-lock.js";
import { detectAndLoadPromptImages } from "./images.js";
import {
readPersistedMediaImageLayout,
readPersistedPromptMediaFacts,
} from "./prompt-image-metadata.js";
import type { EmbeddedRunAttemptParams } from "./types.js";
type PromptExecutionAttempt = Pick<
EmbeddedRunAttemptParams,
"config" | "imageOrder" | "images" | "model" | "sessionFile" | "sessionKey"
| "config"
| "imageOrder"
| "images"
| "media"
| "model"
| "sessionFile"
| "sessionKey"
| "userTurnTranscriptRecorder"
>;
type PromptImageResult = Awaited<ReturnType<typeof detectAndLoadPromptImages>>;
function emptyPromptImages(): PromptImageResult {
return {
images: [],
imageFactIndexes: [],
detectedRefs: [],
failedMediaCount: 0,
loadedCount: 0,
skippedCount: 0,
};
@@ -57,12 +71,24 @@ export async function prepareEmbeddedAttemptPromptExecution(input: {
input.sessionLockController.publishOwnedSessionFileSnapshot(snapshot),
});
const persistedMessage =
attempt.userTurnTranscriptRecorder?.message ??
(await attempt.userTurnTranscriptRecorder?.resolveMessage());
const persistedMedia = persistedMessage
? (readPersistedPromptMediaFacts(persistedMessage) ??
resolveMediaFacts(persistedMessage as unknown as Parameters<typeof resolveMediaFacts>[0]))
: [];
return await detectAndLoadPromptImages({
prompt: input.prompt,
workspaceDir: input.effectiveWorkspace,
model: attempt.model,
existingImages: attempt.images,
imageOrder: attempt.imageOrder,
media: persistedMedia.length > 0 ? persistedMedia : attempt.media,
mediaImageLayout: persistedMessage
? readPersistedMediaImageLayout(persistedMessage)
: undefined,
maxBytes: MAX_IMAGE_BYTES,
maxDimensionPx: resolveImageSanitizationLimits(attempt.config).maxDimensionPx,
workspaceOnly: input.effectiveFsWorkspaceOnly,
@@ -39,6 +39,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
activeContextEngine?: SessionManagerInput["activeContextEngine"];
agentDir: string;
effectiveCwd: string;
effectiveFsWorkspaceOnly: boolean;
effectiveWorkspace: string;
initialSystemPrompt: string;
isRawModelRun: boolean;
@@ -167,6 +168,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
attempt,
computerContextEpoch: input.contextGuards.computerContextEpoch,
effectiveCwd: input.effectiveCwd,
effectiveFsWorkspaceOnly: input.effectiveFsWorkspaceOnly,
effectiveWorkspace: input.effectiveWorkspace,
getPrePromptMessageCount: () => state.prePromptMessageCount,
getPromptCache: () => state.promptCache,
@@ -177,6 +179,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
sessionAgentId: input.sessionManager.sessionAgentId,
sessionManager,
settingsManager,
sandbox: input.transport.sandbox,
});
input.lifecycle.onContextGuardsInstalled(contextGuards.remove);
@@ -28,8 +28,59 @@ type PreparedProviderRuntimePluginHandle = ProviderRuntimePluginHandle & {
prepared: true;
};
export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptParams) {
type AttemptWorkspaceParams = Pick<
EmbeddedRunAttemptParams,
| "agentId"
| "config"
| "cwd"
| "execOverrides"
| "sandboxSessionKey"
| "sessionId"
| "sessionKey"
| "workspaceDir"
>;
/** Resolves the shared workspace and sandbox policy used by native and plugin harnesses. */
export async function resolveAttemptWorkspaceSandbox(params: AttemptWorkspaceParams) {
const resolvedWorkspace = resolveUserPath(params.workspaceDir);
await fs.mkdir(resolvedWorkspace, { recursive: true });
const sandboxSessionKey =
params.sandboxSessionKey?.trim() || params.sessionKey?.trim() || params.sessionId;
const sandbox = await resolveSandboxContext({
config: params.config,
execOverrides: params.execOverrides,
sessionKey: sandboxSessionKey,
workspaceDir: resolvedWorkspace,
});
const effectiveWorkspace =
sandbox?.enabled && sandbox.workspaceAccess !== "rw" ? sandbox.workspaceDir : resolvedWorkspace;
const requestedCwd = params.cwd ? resolveUserPath(params.cwd) : undefined;
if (sandbox?.enabled && requestedCwd && requestedCwd !== resolvedWorkspace) {
throw new Error(
"cwd override is not supported for sandboxed embedded agent runs; omit cwd or use the agent workspace as cwd",
);
}
await fs.mkdir(effectiveWorkspace, { recursive: true });
const { sessionAgentId } = resolveSessionAgentIds({
sessionKey: params.sessionKey,
config: params.config,
agentId: params.agentId,
});
return {
effectiveCwd: sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace),
effectiveFsWorkspaceOnly: resolveAttemptFsWorkspaceOnly({
config: params.config,
sessionAgentId,
}),
effectiveWorkspace,
resolvedWorkspace,
sandbox,
sandboxSessionKey,
sessionAgentId,
};
}
export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptParams) {
// Ultra is a logical orchestration mode, not a provider effort. Preserve it for
// prompt/status surfaces, then lower only at agent-core and provider boundaries.
const agentCoreThinkingLevel = mapThinkingLevel(params.thinkLevel);
@@ -73,28 +124,8 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara
}
};
await fs.mkdir(resolvedWorkspace, { recursive: true });
const sandboxSessionKey =
params.sandboxSessionKey?.trim() || params.sessionKey?.trim() || params.sessionId;
const sandbox = await resolveSandboxContext({
config: params.config,
execOverrides: params.execOverrides,
sessionKey: sandboxSessionKey,
workspaceDir: resolvedWorkspace,
});
const effectiveWorkspace = sandbox?.enabled
? sandbox.workspaceAccess === "rw"
? resolvedWorkspace
: sandbox.workspaceDir
: resolvedWorkspace;
const requestedCwd = params.cwd ? resolveUserPath(params.cwd) : undefined;
if (sandbox?.enabled && requestedCwd && requestedCwd !== resolvedWorkspace) {
throw new Error(
"cwd override is not supported for sandboxed embedded agent runs; omit cwd or use the agent workspace as cwd",
);
}
const effectiveCwd = sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace);
await fs.mkdir(effectiveWorkspace, { recursive: true });
const workspace = await resolveAttemptWorkspaceSandbox(params);
const { effectiveWorkspace } = workspace;
const getCurrentAttemptPluginMetadataSnapshot = (): PluginMetadataSnapshot | undefined =>
params.preparedModelRuntime?.metadataSnapshot;
@@ -141,22 +172,11 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara
};
return providerRuntimeHandle;
};
const { sessionAgentId } = resolveSessionAgentIds({
sessionKey: params.sessionKey,
config: params.config,
agentId: params.agentId,
});
const effectiveFsWorkspaceOnly = resolveAttemptFsWorkspaceOnly({
config: params.config,
sessionAgentId,
});
prepStages.mark("workspace-sandbox");
return {
agentCoreThinkingLevel,
effectiveCwd,
effectiveFsWorkspaceOnly,
effectiveWorkspace,
...workspace,
emitCorePluginToolStageSummary,
emitPrepStageSummary,
getCurrentAttemptPluginMetadataSnapshot,
@@ -164,9 +184,5 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara
prepStages,
proactiveSubagentOrchestration,
providerThinkingLevel,
resolvedWorkspace,
sandbox,
sandboxSessionKey,
sessionAgentId,
};
}
@@ -60,14 +60,18 @@ describe("embedded OpenClaw queued steering cancellation", () => {
{ path: "/tmp/a.png", contentType: "image/png" },
{ path: "/tmp/b.pdf", contentType: "application/pdf" },
];
const imageOrder = ["offloaded", "inline"] as const;
const activeSession: EmbeddedAgentActiveSessionSteerTarget = {
steer,
subscribe: () => () => {},
};
await steerActiveSessionWithOptionalDeliveryWait(activeSession, "inspect both", { media });
await steerActiveSessionWithOptionalDeliveryWait(activeSession, "inspect both", {
media,
imageOrder: [...imageOrder],
});
expect(steer).toHaveBeenCalledWith("inspect both", undefined, undefined, media);
expect(steer).toHaveBeenCalledWith("inspect both", undefined, undefined, media, imageOrder);
});
it("waits for the queued user message_end transcript boundary", async () => {
@@ -4,6 +4,7 @@
import { toErrorObject } from "../../../infra/errors.js";
import type { ImageContent } from "../../../llm/types.js";
import type { MediaFact } from "../../../media/media-facts.js";
import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js";
import type { UserTurnTranscriptRecorder } from "../../../sessions/user-turn-transcript.types.js";
import {
cancelPendingAgentQuestionForSession,
@@ -24,6 +25,7 @@ type EmbeddedAgentActiveSessionSteerTarget = {
images?: ImageContent[],
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder,
media?: MediaFact[],
imageOrder?: PromptImageOrderEntry[],
): Promise<void>;
subscribe(listener: (event: unknown) => void): () => void;
};
@@ -37,9 +39,10 @@ function steerActiveSession(
images?: ImageContent[],
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder,
media?: MediaFact[],
imageOrder?: PromptImageOrderEntry[],
): Promise<void> {
if (media?.length) {
return activeSession.steer(text, images, userTurnTranscriptRecorder, media);
return activeSession.steer(text, images, userTurnTranscriptRecorder, media, imageOrder);
}
return userTurnTranscriptRecorder
? activeSession.steer(text, images, userTurnTranscriptRecorder)
@@ -157,6 +160,7 @@ async function steerAndWaitForTranscriptCommit(
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder,
images?: ImageContent[],
media?: MediaFact[],
imageOrder?: PromptImageOrderEntry[],
): Promise<void> {
await new Promise<void>((resolve, reject) => {
let settled = false;
@@ -243,6 +247,7 @@ async function steerAndWaitForTranscriptCommit(
images,
userTurnTranscriptRecorder,
media,
imageOrder,
);
steer.catch((err: unknown) => {
finish(err);
@@ -291,6 +296,7 @@ export async function steerActiveSessionWithOptionalDeliveryWait(
options?.images,
options?.userTurnTranscriptRecorder,
options?.media,
options?.imageOrder,
);
return;
}
@@ -301,5 +307,6 @@ export async function steerActiveSessionWithOptionalDeliveryWait(
options.userTurnTranscriptRecorder,
options.images,
options.media,
options.imageOrder,
);
}
@@ -216,7 +216,9 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => {
const prepareSessionManagerForRunMock = vi.fn(async (_params?: unknown) => undefined);
const detectAndLoadPromptImagesMock = vi.fn(async () => ({
images: [],
imageFactIndexes: [],
detectedRefs: [],
failedMediaCount: 0,
loadedCount: 0,
skippedCount: 0,
}));
@@ -330,6 +330,7 @@ export async function runEmbeddedAttempt(
...(activeContextEngine ? { activeContextEngine } : {}),
agentDir,
effectiveCwd,
effectiveFsWorkspaceOnly,
effectiveWorkspace,
initialSystemPrompt: preparedSystemPrompt.systemPromptText,
isRawModelRun,
@@ -1,9 +1,18 @@
// History image prune tests keep provider replay compact by replacing stale
// image bytes and media references while preserving recent user context.
// image bytes and fact-owned projections while preserving recent user context.
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import type { AgentMessage } from "openclaw/plugin-sdk/agent-core";
import type { ImageContent } from "openclaw/plugin-sdk/llm";
import { describe, expect, it } from "vitest";
import {
attachRuntimePromptMediaFacts,
readRuntimePromptMediaFacts,
} from "../../../media/media-facts.js";
import { buildPersistedUserTurnMessage } from "../../../sessions/user-turn-transcript.js";
import { castAgentMessage } from "../../test-helpers/agent-message-fixtures.js";
import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js";
import {
installHistoryImagePruneContextTransform,
pruneProcessedHistoryImages,
@@ -12,6 +21,8 @@ import {
const PRUNED_HISTORY_IMAGE_MARKER = "[image data removed - already processed by model]";
const PRUNED_HISTORY_MEDIA_REFERENCE_MARKER =
"[media reference removed - already processed by model]";
const TINY_PNG_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
function expectArrayMessageContent(
message: AgentMessage | undefined,
@@ -105,9 +116,23 @@ describe("pruneProcessedHistoryImages", () => {
expect(content[0]?.type).toBe("text");
});
it("scrubs old media attachment markers from text blocks", () => {
// Text references are scrubbed alongside image blocks so old paths and
// media URIs cannot rehydrate stale images on a later replay.
it("strips explicit-image provenance when its old image block is pruned", () => {
const message = castAgentMessage({
role: "user",
content: [{ type: "text", text: "explicit image" }, { ...image }],
__openclaw: {
mediaImageBlockFactIndexes: [null],
mediaImageLayout: { slots: [{ kind: "inline" }] },
},
});
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const meta = (pruned[0] as unknown as Record<string, unknown>)["__openclaw"];
expect(meta).toBeUndefined();
});
it("redacts factless legacy attachment text while pruning old image blocks", () => {
const messages: AgentMessage[] = [
castAgentMessage({
role: "user",
@@ -147,7 +172,7 @@ describe("pruneProcessedHistoryImages", () => {
expectContentBlock(originalContent[1], { type: "image", data: "abc" });
});
it("scrubs old media attachment markers from string content without image blocks", () => {
it("redacts factless legacy attachment text without image blocks", () => {
const messages: AgentMessage[] = [
castAgentMessage({
role: "user",
@@ -157,16 +182,11 @@ describe("pruneProcessedHistoryImages", () => {
];
const pruned = expectPrunedMessages(messages);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe(`please remember ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}`);
const originalUser = messages[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(originalUser?.content).toBe(
"please remember [media attached: media://inbound/stale-image.png]",
);
const user = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(user?.content).toBe(`please remember ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}`);
});
it("prunes marked bare late-media turns identically to legacy literal turns", () => {
it("prunes fact-owned and factless legacy late-media projections", () => {
const fields = {
role: "user" as const,
MediaPath: "media://inbound/stale-image.png",
@@ -178,8 +198,8 @@ describe("pruneProcessedHistoryImages", () => {
__openclaw: { lateMedia: true },
});
const legacyString = castAgentMessage({
...fields,
content: "[media attached: media://inbound/stale-image.png]",
role: "user",
});
const markedArray = castAgentMessage({
...fields,
@@ -187,7 +207,7 @@ describe("pruneProcessedHistoryImages", () => {
__openclaw: { lateMedia: true },
});
const legacyArray = castAgentMessage({
...fields,
role: "user",
content: [
{ type: "text", text: "[media attached: media://inbound/stale-image.png]" },
{ ...image },
@@ -199,12 +219,20 @@ describe("pruneProcessedHistoryImages", () => {
const prunedMarkedArray = expectPrunedMessages([markedArray, ...oldEnoughTail()]);
const prunedLegacyArray = expectPrunedMessages([legacyArray, ...oldEnoughTail()]);
const markedStringOutput = prunedMarkedString as unknown as Array<{ content?: unknown }>;
const legacyStringOutput = prunedLegacyString as unknown as Array<{ content?: unknown }>;
const markedArrayOutput = prunedMarkedArray as unknown as Array<{ content?: unknown }>;
const legacyStringOutput = prunedLegacyString as unknown as Array<{ content?: unknown }>;
const legacyArrayOutput = prunedLegacyArray as unknown as Array<{ content?: unknown }>;
expect(markedStringOutput[0]?.content).toBe(legacyStringOutput[0]?.content);
expect(markedArrayOutput[0]?.content).toEqual(legacyArrayOutput[0]?.content);
expect(markedStringOutput[0]?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
expect(legacyStringOutput[0]?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
expect(markedArrayOutput[0]?.content).toEqual([
{ type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER },
{ type: "text", text: PRUNED_HISTORY_IMAGE_MARKER },
]);
expect(legacyArrayOutput[0]?.content).toEqual([
{ type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER },
{ type: "text", text: PRUNED_HISTORY_IMAGE_MARKER },
]);
});
it("does not replace a distinct caption on an old marked late-media turn", () => {
@@ -216,12 +244,178 @@ describe("pruneProcessedHistoryImages", () => {
__openclaw: { lateMedia: true },
});
expect(pruneProcessedHistoryImages([message, ...oldEnoughTail()])).toBeNull();
const output = [message] as unknown as Array<{ content?: unknown }>;
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const output = pruned as unknown as Array<{ content?: unknown; MediaPath?: unknown }>;
expect(output[0]?.content).toBe("resolved subtitle");
expect(output[0]?.MediaPath).toBeUndefined();
});
it("scrubs bare old inbound media URIs from tool results", () => {
it("drops runtime facts from captioned old turns without changing their text", () => {
const message = attachRuntimePromptMediaFacts(
castAgentMessage({ role: "user", content: "caption stays byte-identical" }),
[{ path: "/tmp/stale.png", contentType: "image/png" }],
);
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe("caption stays byte-identical");
expect(firstUser && readRuntimePromptMediaFacts(firstUser)).toBeUndefined();
expect(readRuntimePromptMediaFacts(message)).toHaveLength(1);
});
it("redacts the exact fact-owned projection from a captioned old turn", () => {
const imagePath = "/tmp/stale-owned.png";
const message = attachRuntimePromptMediaFacts(
castAgentMessage({
role: "user",
content: `[media attached: ${imagePath} (image/png)]\n\ncaption stays`,
}),
[{ path: imagePath, contentType: "image/png" }],
);
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe(`${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}\n\ncaption stays`);
});
it("redacts a persisted fact projection with a distinct URL", () => {
const imagePath = "/tmp/stale-owned.png";
const imageUrl = "https://example.test/stale-owned.png";
const message = castAgentMessage({
role: "user",
content: `[media attached: ${imagePath} (image/png) | ${imageUrl}]`,
__openclaw: {
media: [
{
path: imagePath,
url: imageUrl,
contentType: "image/png",
hydrationSuppressed: true,
},
],
},
});
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
});
it("redacts a persisted relative projection after legacy path canonicalization", () => {
const message = buildPersistedUserTurnMessage({
text: "[media attached: ./old.png (image/png)]",
media: [
{
path: "./old.png",
workspaceDir: "/workspace",
contentType: "image/png",
},
],
}) as AgentMessage;
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
});
it("redacts a persisted relative projection without a dot prefix", () => {
const message = buildPersistedUserTurnMessage({
text: "[media attached: sub/old.png (image/png)]",
media: [
{
path: "sub/old.png",
workspaceDir: "/workspace",
contentType: "image/png",
},
],
}) as AgentMessage;
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
});
it("does not claim a basename-only marker for an absolute owned fact", () => {
const message = buildPersistedUserTurnMessage({
text: "caption mentions [media attached: ./photo.png (image/png)]",
media: [{ path: "/workspace/sub/photo.png", contentType: "image/png" }],
}) as AgentMessage;
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe("caption mentions [media attached: ./photo.png (image/png)]");
});
it("preserves an unrelated legacy-shaped marker in a fact-backed caption", () => {
const message = buildPersistedUserTurnMessage({
text: "caption mentions [media attached: ./unrelated.png (image/png)]",
media: [{ path: "./owned.png", workspaceDir: "/workspace", contentType: "image/png" }],
}) as AgentMessage;
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe(
"caption mentions [media attached: ./unrelated.png (image/png)]",
);
});
it("redacts separately projected fact lines in distinct text blocks", () => {
const firstPath = "/tmp/first-owned.png";
const secondPath = "/tmp/second-owned.png";
const message = attachRuntimePromptMediaFacts(
castAgentMessage({
role: "user",
content: [
{ type: "text", text: `[media attached: ${firstPath} (image/png)]` },
{ type: "text", text: `[media attached: ${secondPath} (image/png)]` },
],
}),
[
{ path: firstPath, contentType: "image/png" },
{ path: secondPath, contentType: "image/png" },
],
);
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
expect(expectArrayMessageContent(pruned[0], "expected redacted text blocks")).toEqual([
{ type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER },
{ type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER },
]);
});
it("redacts a fact-backed legacy image-source projection", () => {
const imagePath = "/tmp/legacy-owned.jpg";
const message = castAgentMessage({
role: "user",
content: `[Image: source: ${imagePath}]\ncaption stays`,
MediaPath: imagePath,
MediaPaths: [imagePath],
MediaType: "image/jpeg",
MediaTypes: ["image/jpeg"],
});
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe(`${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}\ncaption stays`);
});
it("preserves a bare claim-check URI in a fact-backed caption", () => {
const mediaRef = "media://inbound/captioned.png";
const message = castAgentMessage({
role: "user",
content: `caption mentions ${mediaRef} as text`,
MediaPath: mediaRef,
MediaPaths: [mediaRef],
MediaType: "image/png",
MediaTypes: ["image/png"],
});
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
expect(firstUser?.content).toBe(`caption mentions ${mediaRef} as text`);
});
it("redacts bare old inbound media URIs from factless tool results", () => {
const messages: AgentMessage[] = [
castAgentMessage({
role: "toolResult",
@@ -232,15 +426,8 @@ describe("pruneProcessedHistoryImages", () => {
];
const pruned = expectPrunedMessages(messages);
const toolResult = pruned[0] as Extract<AgentMessage, { role: "toolResult" }> | undefined;
expect(toolResult?.content).toBe(`previous ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER} result`);
const originalToolResult = messages[0] as
| Extract<AgentMessage, { role: "toolResult" }>
| undefined;
expect(originalToolResult?.content).toBe(
"previous media://inbound/stale-screenshot.png result",
);
});
it("keeps image blocks that belong to the third-most-recent assistant turn", () => {
@@ -453,4 +640,133 @@ describe("installHistoryImagePruneContextTransform", () => {
restore();
expect(agent.transformContext).toBe(originalTransformContext);
});
it("does not legacy-redact a fact-backed caption on the wrapper second pass", async () => {
const mediaRef = "media://inbound/captioned.png";
const messages: AgentMessage[] = [
castAgentMessage({
role: "user",
content: `caption mentions ${mediaRef} as text`,
media: [{ url: mediaRef, contentType: "image/png" }],
}),
...oldEnoughTail(),
];
const agent = {
transformContext: async (input: AgentMessage[]) =>
input.map((message) => ({ ...message })) as AgentMessage[],
};
const restore = installHistoryImagePruneContextTransform(agent);
const replay = await agent.transformContext(messages);
expect((replay[0] as { content?: unknown }).content).toBe(
`caption mentions ${mediaRef} as text`,
);
restore();
});
it("hydrates recent facts before an existing transform clones messages", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-history-hydrate-"));
const imagePath = path.join(workspaceDir, "photo.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const message = attachRuntimePromptMediaFacts(
castAgentMessage({ role: "user", content: [{ type: "text", text: "describe" }] }),
[{ path: imagePath, contentType: "image/png" }],
);
const agent = {
transformContext: async (messages: AgentMessage[]) =>
messages.map((entry) => {
if (!("content" in entry)) {
return { ...entry };
}
return {
...entry,
content: Array.isArray(entry.content)
? entry.content.map((block: (typeof entry.content)[number]) => ({ ...block }))
: entry.content,
};
}) as AgentMessage[],
};
const restore = installHistoryImagePruneContextTransform(agent, {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
try {
const replay = await agent.transformContext([message]);
expect(expectArrayMessageContent(replay[0], "expected hydrated content")).toEqual([
{ type: "text", text: "describe" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
} finally {
restore();
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("strips nested media metadata before old turns can rehydrate", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-pruned-nested-media-"));
const imagePath = path.join(workspaceDir, "old.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const baseBridge = createHostSandboxFsBridge(workspaceDir);
let hydrationReadCount = 0;
const bridge = {
...baseBridge,
readFile: async (params: Parameters<typeof baseBridge.readFile>[0]) => {
hydrationReadCount++;
return await baseBridge.readFile(params);
},
};
const message = castAgentMessage({
role: "user",
content: "[media attached: ./old.png (image/png)]",
__openclaw: {
media: [{ path: "./old.png", contentType: "image/png" }],
mediaImageBlockFactIndexes: [0],
mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] },
},
});
const agent: {
transformContext?: (messages: AgentMessage[]) => Promise<AgentMessage[]> | AgentMessage[];
} = {};
const restore = installHistoryImagePruneContextTransform(agent, {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
sandbox: { root: workspaceDir, bridge },
});
try {
const replay = await agent.transformContext?.([message, ...oldEnoughTail()]);
const meta = (replay?.[0] as unknown as Record<string, unknown>)?.["__openclaw"] as
| Record<string, unknown>
| undefined;
expect(hydrationReadCount).toBe(0);
expect(meta?.media).toBeUndefined();
expect(meta?.mediaImageBlockFactIndexes).toBeUndefined();
expect(meta?.mediaImageLayout).toBeUndefined();
} finally {
restore();
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("treats identity-less nested facts as structured ownership", () => {
const message = castAgentMessage({
role: "user",
content: "[media attached: /tmp/unknown.png (image/png)]",
__openclaw: {
media: [{ kind: "image" }],
mediaImageLayout: { slots: [], suppressedFactIndexes: [0] },
},
});
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
const first = pruned[0] as unknown as Record<string, unknown>;
const meta = first["__openclaw"] as Record<string, unknown> | undefined;
expect(first.content).toBe("[media attached: /tmp/unknown.png (image/png)]");
expect(meta?.media).toBeUndefined();
expect(meta?.mediaImageLayout).toBeUndefined();
});
});
@@ -1,20 +1,31 @@
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
import {
normalizeMediaFacts,
projectMediaFacts,
readRuntimePromptMediaFacts,
resolveMediaFacts,
type MediaFact,
} from "../../../media/media-facts.js";
/**
* Prunes already-processed image payloads from replayed prompt history.
*/
import { buildLateMediaAttachedProjection } from "../../../sessions/user-turn-transcript.js";
import type { AgentMessage } from "../../runtime/index.js";
import { hasNonBlankUserText } from "./attempt.user-message-boundary.js";
import { hydratePromptMediaMessages } from "./images.js";
/** Replacement text for old image blocks that were already available to the model. */
const PRUNED_HISTORY_IMAGE_MARKER = "[image data removed - already processed by model]";
/** Replacement text for old textual media references that would otherwise be reloaded. */
/** Replacement text for fact-owned late-media projections already processed by the model. */
const PRUNED_HISTORY_MEDIA_REFERENCE_MARKER =
"[media reference removed - already processed by model]";
const MEDIA_ATTACHED_HISTORY_REF_PATTERN = /\[media attached(?:\s+\d+\/\d+)?:\s*[^\]]+\]/gi;
const MESSAGE_IMAGE_HISTORY_REF_PATTERN = /\[Image:\s*source:\s*[^\]]+\]/gi;
const INBOUND_MEDIA_URI_HISTORY_REF_PATTERN = /\bmedia:\/\/inbound\/[^\]\s/\\]+/g;
// Legacy replay hygiene only: factless pre-MediaFact rows past the prune cutoff
// retain no attachment ownership. Fact-bearing messages never use these patterns.
const LEGACY_MEDIA_ATTACHED_PATTERN = /\[media attached(?:\s+\d+\/\d+)?:\s*[^\]]+\]/gi;
const LEGACY_IMAGE_SOURCE_PATTERN = /\[Image:\s*source:\s*[^\]]+\]/gi;
const LEGACY_INBOUND_MEDIA_URI_PATTERN = /\bmedia:\/\/inbound\/[^\]\s/\\]+/g;
type PrunableContextAgent = {
transformContext?: (
@@ -29,6 +40,17 @@ type PrunableContextAgent = {
* ones, so text-only turns consume the window.
*/
const PRESERVE_RECENT_COMPLETED_TURNS = 3;
const PERSISTED_MEDIA_FIELD_KEYS = [
"media",
"MediaPath",
"MediaPaths",
"MediaUrl",
"MediaUrls",
"MediaType",
"MediaTypes",
"MediaTranscribedIndexes",
"MediaWorkspaceDir",
] as const;
function resolvePruneBeforeIndex(messages: AgentMessage[]): number {
const completedTurnStarts: number[] = [];
@@ -66,18 +88,133 @@ function resolvePruneBeforeIndex(messages: AgentMessage[]): number {
return completedTurnStarts.at(-PRESERVE_RECENT_COMPLETED_TURNS) ?? -1;
}
function pruneHistoryMediaReferenceText(text: string): string {
function resolveMessageMediaFacts(message: AgentMessage): MediaFact[] {
const runtimeMedia = readRuntimePromptMediaFacts(message);
if (runtimeMedia) {
return runtimeMedia;
}
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
const nestedMedia =
meta && typeof meta === "object" && !Array.isArray(meta)
? (meta as Record<string, unknown>).media
: undefined;
return Array.isArray(nestedMedia)
? normalizeMediaFacts(nestedMedia as MediaFact[])
: resolveMediaFacts(message as unknown as Parameters<typeof resolveMediaFacts>[0]);
}
function wasStructurallyMediaPruned(message: AgentMessage): boolean {
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
return (
Boolean(meta) &&
typeof meta === "object" &&
!Array.isArray(meta) &&
(meta as Record<string, unknown>).mediaImagePruned === true
);
}
function replaceLegacyFactlessMediaText(text: string): string {
return text
.replace(MEDIA_ATTACHED_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
.replace(MESSAGE_IMAGE_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
.replace(INBOUND_MEDIA_URI_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
.replace(LEGACY_MEDIA_ATTACHED_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
.replace(LEGACY_IMAGE_SOURCE_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
.replace(LEGACY_INBOUND_MEDIA_URI_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
}
function factOwnsMarkerIdentity(identity: string, media: MediaFact[]): boolean {
const normalizedIdentity = identity.replaceAll("\\", "/");
return media.some((fact) =>
[fact.path, fact.url].some((alias) => alias?.replaceAll("\\", "/") === normalizedIdentity),
);
}
function extractMediaAttachedIdentity(marker: string): string {
const content = marker.replace(/^\[media attached(?:\s+\d+\/\d+)?:\s*/i, "").slice(0, -1);
const mimeIndex = content.lastIndexOf(" (");
const urlIndex = content.indexOf(" | ");
const endIndexes = [mimeIndex, urlIndex].filter((index) => index >= 0);
const endIndex = endIndexes.length > 0 ? Math.min(...endIndexes) : content.length;
return content.slice(0, endIndex).trim();
}
function replaceOwnedLegacyMediaMarkers(text: string, media: MediaFact[]): string {
return text
.replace(LEGACY_MEDIA_ATTACHED_PATTERN, (marker) =>
factOwnsMarkerIdentity(extractMediaAttachedIdentity(marker), media)
? PRUNED_HISTORY_MEDIA_REFERENCE_MARKER
: marker,
)
.replace(LEGACY_IMAGE_SOURCE_PATTERN, (marker) => {
const identity = marker
.replace(/^\[Image:\s*source:\s*/i, "")
.slice(0, -1)
.trim();
return factOwnsMarkerIdentity(identity, media)
? PRUNED_HISTORY_MEDIA_REFERENCE_MARKER
: marker;
});
}
function replaceOwnedMediaProjection(text: string, media: MediaFact[]): string {
if (media.length === 0) {
return text;
}
const projectionLines = new Set<string>();
for (const facts of [media, ...media.map((fact) => [fact])]) {
const projection = buildInboundMediaNoteProjection({
media: facts,
...projectMediaFacts(facts, "channel"),
}).text;
for (const line of projection?.split("\n") ?? []) {
if (line) {
projectionLines.add(line);
}
}
}
let redacted = text;
for (const line of projectionLines) {
redacted = redacted.replaceAll(line, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
}
for (const fact of media) {
for (const alias of [fact.path, fact.url].filter((value): value is string => Boolean(value))) {
redacted = redacted
.replaceAll(`[Image: source: ${alias}]`, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
.replaceAll(`[media attached: ${alias}]`, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
}
}
return replaceOwnedLegacyMediaMarkers(redacted, media);
}
function cloneMessageWithContent(
message: Extract<AgentMessage, { role: "user" | "toolResult" }>,
content: typeof message.content,
dropMedia = false,
dropImageMetadata = dropMedia,
): AgentMessage {
return { ...message, content } as AgentMessage;
const clone = { ...message, content } as AgentMessage & Record<string, unknown>;
if (dropMedia) {
for (const key of PERSISTED_MEDIA_FIELD_KEYS) {
delete clone[key];
}
}
if (dropImageMetadata) {
const meta = clone["__openclaw"];
const nextMeta =
meta && typeof meta === "object" && !Array.isArray(meta)
? { ...(meta as Record<string, unknown>) }
: {};
delete nextMeta.mediaImageBlockFactIndexes;
delete nextMeta.mediaImageLayout;
if (dropMedia) {
delete nextMeta.media;
nextMeta.mediaImagePruned = true;
}
if (Object.keys(nextMeta).length > 0) {
clone["__openclaw"] = nextMeta;
} else {
delete clone["__openclaw"];
}
}
return clone;
}
/** Prunes old image payloads and references before later LLM-boundary synthesis. */
@@ -93,12 +230,18 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess
if (!message || (message.role !== "user" && message.role !== "toolResult")) {
continue;
}
const media = message.role === "user" ? resolveMessageMediaFacts(message) : [];
const hasOwnedMedia = media.length > 0;
const structuredMediaWasPruned = wasStructurallyMediaPruned(message);
// Materialize blank marked turns here so this earlier boundary still prunes stale paths.
const lateMediaText =
const lateMediaProjection =
message.role === "user" && !hasNonBlankUserText(message.content)
? buildLateMediaAttachedProjection(message).text
? buildLateMediaAttachedProjection(message)
: undefined;
const lateMediaText = lateMediaProjection?.media
.map(() => PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
.join("\n");
const content = lateMediaText
? Array.isArray(message.content)
? ([{ type: "text", text: lateMediaText }, ...message.content] as typeof message.content)
@@ -106,10 +249,14 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess
: message.content;
if (typeof content === "string") {
const prunedText = pruneHistoryMediaReferenceText(content);
if (prunedText !== message.content) {
const nextText = hasOwnedMedia
? replaceOwnedMediaProjection(content, media)
: structuredMediaWasPruned
? content
: replaceLegacyFactlessMediaText(content);
if (nextText !== message.content || hasOwnedMedia) {
prunedMessages ??= messages.slice();
prunedMessages[i] = cloneMessageWithContent(message, prunedText);
prunedMessages[i] = cloneMessageWithContent(message, nextText, hasOwnedMedia);
}
continue;
}
@@ -121,16 +268,34 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess
const nextContent = content.map((block) => {
const typed = block as { type?: unknown; text?: unknown } | null | undefined;
if (typed?.type === "text" && typeof typed.text === "string") {
const text = pruneHistoryMediaReferenceText(typed.text);
return text === typed.text ? block : ({ ...block, text } as (typeof content)[number]);
const text = hasOwnedMedia
? replaceOwnedMediaProjection(typed.text, media)
: structuredMediaWasPruned
? typed.text
: replaceLegacyFactlessMediaText(typed.text);
if (text !== typed.text) {
return { ...block, text } as (typeof content)[number];
}
}
return typed?.type === "image"
? ({ type: "text", text: PRUNED_HISTORY_IMAGE_MARKER } as (typeof content)[number])
: block;
});
if (lateMediaText || nextContent.some((block, index) => block !== content[index])) {
const prunedImageBlock = content.some(
(block) => (block as { type?: unknown } | null | undefined)?.type === "image",
);
if (
hasOwnedMedia ||
lateMediaText ||
nextContent.some((block, index) => block !== content[index])
) {
prunedMessages ??= messages.slice();
prunedMessages[i] = cloneMessageWithContent(message, nextContent);
prunedMessages[i] = cloneMessageWithContent(
message,
nextContent,
hasOwnedMedia,
hasOwnedMedia || prunedImageBlock,
);
}
}
@@ -138,13 +303,20 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess
}
/** Installs an agent context transform that prunes old image/media history before model input. */
export function installHistoryImagePruneContextTransform(agent: PrunableContextAgent): () => void {
export function installHistoryImagePruneContextTransform(
agent: PrunableContextAgent,
mediaOptions?: Parameters<typeof hydratePromptMediaMessages>[1],
): () => void {
const originalTransformContext = agent.transformContext;
agent.transformContext = async (messages: AgentMessage[], signal?: AbortSignal) => {
const prunedInput = pruneProcessedHistoryImages(messages) ?? messages;
const hydratedInput = mediaOptions
? await hydratePromptMediaMessages(prunedInput, mediaOptions)
: prunedInput;
const transformed = originalTransformContext
? await originalTransformContext.call(agent, messages, signal)
: messages;
const sourceMessages = Array.isArray(transformed) ? transformed : messages;
? await originalTransformContext.call(agent, hydratedInput, signal)
: hydratedInput;
const sourceMessages = Array.isArray(transformed) ? transformed : hydratedInput;
return pruneProcessedHistoryImages(sourceMessages) ?? sourceMessages;
};
return () => {
@@ -0,0 +1,157 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { createSolidPngBuffer } from "../../../../test/helpers/image-fixtures.js";
import { detectAndLoadPromptImages } from "./images.js";
const HYDRATION_PARTS = ["inline", "offloaded", "suppressed", "explicit", "legacy"] as const;
type HydrationPart = (typeof HYDRATION_PARTS)[number];
type HydrationCombination = { name: string; parts: readonly HydrationPart[] };
function combinations<T>(values: readonly T[], size: number): T[][] {
if (size === 0) {
return [[]];
}
return values.flatMap((value, index) =>
combinations(values.slice(index + 1), size - 1).map((tail) => [value].concat(tail)),
);
}
const HYDRATION_COMBINATIONS: HydrationCombination[] = [2, 3].flatMap((size) =>
combinations(HYDRATION_PARTS, size).map((parts) => ({
name: parts.join(" + "),
parts,
})),
);
describe("hydration combination matrix", () => {
it("attributes a suppressed-plus-inline sanitization failure to the inline fact", async () => {
const result = await detectAndLoadPromptImages({
prompt: "already described",
media: [
{
path: "/tmp/described-missing.png",
contentType: "image/png",
hydrationSuppressed: true,
},
{ path: "/tmp/inline.png", contentType: "image/png" },
],
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }],
imageOrder: ["inline"],
});
expect(result.images).toEqual([]);
expect(result.imageFactIndexes).toEqual([]);
expect(result.loadedCount).toBe(0);
expect(result.failedMediaCount).toBe(1);
});
it.each(HYDRATION_COMBINATIONS)(
"$name preserves materialization, order, and suppression invariants",
async (testCase: HydrationCombination) => {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-hydration-matrix-"));
const has = (part: HydrationPart) => testCase.parts.includes(part);
const inlineBuffer = createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 });
const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 });
const explicitBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 });
const inlinePath = path.join(root, "inline.png");
const offloadedPath = path.join(root, "offloaded.png");
const explicitPath = path.join(root, "explicit.png");
const suppressedPath = path.join(root, "suppressed-missing.png");
if (has("offloaded")) {
await fs.writeFile(offloadedPath, offloadedBuffer);
}
if (has("explicit")) {
await fs.writeFile(explicitPath, explicitBuffer);
}
const media: Array<{
path: string;
contentType: string;
hydrationSuppressed?: boolean;
}> = [];
if (has("suppressed")) {
media.push({
path: suppressedPath,
contentType: "image/png",
hydrationSuppressed: true,
});
}
const inlineFactIndex = has("inline") ? media.length : undefined;
if (has("inline")) {
media.push({ path: inlinePath, contentType: "image/png" });
}
const offloadedFactIndex = has("offloaded") ? media.length : undefined;
if (has("offloaded")) {
media.push({ path: offloadedPath, contentType: "image/png" });
}
const inlineImage = {
type: "image" as const,
data: inlineBuffer.toString("base64"),
mimeType: "image/png",
};
const imageOrder = [
...(has("inline") ? (["inline"] as const) : []),
...(has("offloaded") ? (["offloaded"] as const) : []),
];
const existingImages = has("inline") ? [inlineImage] : undefined;
const existingImageFactIndexes =
has("legacy") && inlineFactIndex !== undefined ? [inlineFactIndex] : undefined;
try {
const result = await detectAndLoadPromptImages({
prompt: has("explicit") ? `inspect ${explicitPath}` : "inspect attachments",
media,
workspaceDir: root,
model: { input: ["text", "image"] },
existingImages,
existingImageFactIndexes,
imageOrder: has("legacy") ? undefined : imageOrder,
workspaceOnly: true,
});
const expectedImages = [
...(has("inline") ? [inlineImage] : []),
...(has("offloaded")
? [
{
type: "image" as const,
data: offloadedBuffer.toString("base64"),
mimeType: "image/png",
},
]
: []),
...(has("explicit")
? [
{
type: "image" as const,
data: explicitBuffer.toString("base64"),
mimeType: "image/png",
},
]
: []),
];
const expectedFactIndexes = [
...(inlineFactIndex === undefined ? [] : [inlineFactIndex]),
...(offloadedFactIndex === undefined ? [] : [offloadedFactIndex]),
...(has("explicit") ? [null] : []),
];
const expectedLoadedCount = Number(has("offloaded")) + Number(has("explicit"));
expect(result.images).toEqual(expectedImages);
expect(result.imageFactIndexes).toEqual(expectedFactIndexes);
expect(result.loadedCount).toBe(expectedLoadedCount);
expect(result.failedMediaCount).toBe(0);
expect(result.images).toHaveLength(
Number(has("inline")) + Number(has("offloaded")) + Number(has("explicit")),
);
} finally {
await fs.rm(root, { recursive: true, force: true });
}
},
);
});
@@ -0,0 +1,293 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { describe, expect, it } from "vitest";
import { normalizeMediaFacts, resolveMediaFacts } from "../../../media/media-facts.js";
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js";
import { detectAndLoadPromptImages, hasHydratableMediaImages } from "./images.js";
const TINY_PNG_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
describe("fact-carried image references", () => {
it("counts only facts that will hydrate an image attachment", () => {
expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", kind: "image" }])).toBe(true);
// Legacy transcript projections persist bare kinds as the media type.
expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", contentType: "image" }])).toBe(true);
expect(hasHydratableMediaImages([{ path: "/tmp/anim.webp", contentType: "sticker" }])).toBe(
true,
);
expect(hasHydratableMediaImages([{ kind: "image" }])).toBe(false);
expect(
hasHydratableMediaImages([{ url: "https://example.test/remote.png", kind: "image" }]),
).toBe(false);
expect(
hasHydratableMediaImages([{ path: "https://example.test/remote.png", kind: "image" }]),
).toBe(false);
expect(hasHydratableMediaImages([])).toBe(false);
expect(hasHydratableMediaImages(undefined)).toBe(false);
for (const contentType of ["audio", "video", "document"]) {
expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", contentType }])).toBe(false);
}
});
it("retains described-image suppression across fact copy boundaries", async () => {
const normalized = normalizeMediaFacts([
{
path: "/tmp/described.png",
contentType: "image/png",
hydrationSuppressed: true,
},
]);
const reprojected = resolveMediaFacts({
media: normalized,
MediaPaths: ["/tmp/stale.png"],
MediaTypes: ["application/octet-stream"],
});
expect(reprojected).toEqual([
expect.objectContaining({
path: "/tmp/described.png",
contentType: "image/png",
kind: "image",
hydrationSuppressed: true,
}),
]);
const result = await detectAndLoadPromptImages({
prompt: "already described",
media: reprojected,
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
});
expect(result.failedMediaCount).toBe(0);
expect(result.images).toEqual([]);
});
it("pairs identity-less facts with existing inline images when order metadata is absent", async () => {
const existingImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const result = await detectAndLoadPromptImages({
prompt: "look",
media: [{ kind: "image", contentType: "image/png" }],
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
existingImages: [existingImage],
});
expect(result.failedMediaCount).toBe(0);
expect(result.images).toEqual([existingImage]);
});
it("loads an explicit ref matching a fact sliced into an inline slot", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-inline-explicit-ref-"));
const imagePath = path.join(workspaceDir, "photo.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
try {
const result = await detectAndLoadPromptImages({
prompt: `compare ${imagePath}`,
media: [{ path: imagePath, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
existingImages: [inlineImage],
imageOrder: ["inline"],
workspaceOnly: true,
});
expect(result.loadedCount).toBe(1);
expect(result.failedMediaCount).toBe(0);
expect(result.images).toEqual([
inlineImage,
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("keeps identity-bearing refs when image order metadata has more inline slots", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-inline-order-"));
const imagePath = path.join(workspaceDir, "offloaded.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const firstInline = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const secondInline = { ...firstInline };
try {
const result = await detectAndLoadPromptImages({
prompt: "compare",
media: [{ path: imagePath, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
existingImages: [firstInline, secondInline],
imageOrder: ["inline", "inline"],
workspaceOnly: true,
});
expect(result.loadedCount).toBe(1);
expect(result.failedMediaCount).toBe(0);
expect(result.images).toEqual([
firstInline,
secondInline,
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
expect(result.imageFactIndexes).toEqual([null, null, 0]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("fails an exact inline slot whose image block is missing", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-missing-inline-"));
const imagePath = path.join(workspaceDir, "stale.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
try {
const result = await detectAndLoadPromptImages({
prompt: "inspect",
media: [{ path: imagePath, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
imageOrder: ["inline"],
workspaceOnly: true,
});
expect(result.loadedCount).toBe(0);
expect(result.failedMediaCount).toBe(1);
expect(result.images).toEqual([]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("hydrates a fact whose only local identity is a file URL", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-file-url-"));
const imagePath = path.join(workspaceDir, "photo.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
try {
const result = await detectAndLoadPromptImages({
prompt: "",
media: [{ path: pathToFileURL(imagePath).href, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect(result.images).toEqual([
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("hydrates managed inbound media URIs before workspace path resolution", async () => {
// Managed media URIs are canonical inbound attachment handles and should
// work even when workspaceOnly would reject ordinary outside paths.
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-uri-"));
const workspaceDir = path.join(stateDir, "workspace-agent");
const inboundDir = path.join(stateDir, "media", "inbound");
const mediaId = "telegram-photo.png";
await fs.mkdir(workspaceDir, { recursive: true });
await fs.mkdir(inboundDir, { recursive: true });
await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64"));
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
try {
const result = await detectAndLoadPromptImages({
prompt: "",
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
const image = result.images[0];
expect(image?.type).toBe("image");
expect(image?.mimeType).toBe("image/png");
expect(image?.data).toBe(TINY_PNG_BASE64);
} finally {
envSnapshot.restore();
await fs.rm(stateDir, { recursive: true, force: true });
}
});
it("hydrates sandbox-staged inbound media URIs", async () => {
const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-sbx-uri-"));
const inboundDir = path.join(sandboxRoot, "media", "inbound");
const mediaId = "telegram-photo.png";
await fs.mkdir(inboundDir, { recursive: true });
await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64"));
try {
const result = await detectAndLoadPromptImages({
prompt: "",
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
model: { input: ["text", "image"] },
workspaceDir: sandboxRoot,
workspaceOnly: true,
sandbox: {
root: sandboxRoot,
bridge: createHostSandboxFsBridge(sandboxRoot),
},
});
const image = result.images[0];
expect(image?.type).toBe("image");
expect(image?.mimeType).toBe("image/png");
expect(image?.data).toBe(TINY_PNG_BASE64);
} finally {
await fs.rm(sandboxRoot, { recursive: true, force: true });
}
});
it.each([
["traversal", "media://inbound/../secret.png"],
["encoded traversal", "media://inbound/%2e%2e%2fsecret.png"],
["null byte", "media://inbound/secret%00.png"],
])("rejects %s claim-check facts", async (_label: string, mediaUrl: string) => {
const result = await detectAndLoadPromptImages({
prompt: "legacy ticket is carried structurally",
media: [{ url: mediaUrl, contentType: "image/png" }],
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
});
expect(result.loadedCount).toBe(0);
expect(result.skippedCount).toBe(1);
expect(result.images).toHaveLength(0);
});
it("allows sandbox-validated host paths outside default media roots", async () => {
const homeDir = os.homedir();
await fs.mkdir(homeDir, { recursive: true });
const sandboxParent = await fs.mkdtemp(path.join(homeDir, "openclaw-sandbox-image-"));
try {
const sandboxRoot = path.join(sandboxParent, "sandbox");
await fs.mkdir(sandboxRoot, { recursive: true });
const imagePath = path.join(sandboxRoot, "photo.png");
const pngB64 = TINY_PNG_BASE64;
await fs.writeFile(imagePath, Buffer.from(pngB64, "base64"));
const result = await detectAndLoadPromptImages({
prompt: "",
media: [{ path: "./photo.png", contentType: "image/png" }],
model: { input: ["text", "image"] },
workspaceDir: sandboxRoot,
sandbox: {
root: sandboxRoot,
bridge: createHostSandboxFsBridge(sandboxRoot),
},
});
const image = result.images[0];
expect(image?.type).toBe("image");
expect(image?.mimeType).toBe("image/png");
expect(image?.data).toBe(TINY_PNG_BASE64);
} finally {
await fs.rm(sandboxParent, { recursive: true, force: true });
}
});
});
@@ -0,0 +1,160 @@
import { safeFileURLToPath } from "../../../infra/local-file-access.js";
import {
isImageMediaFact,
normalizeMediaFacts,
type MediaFact,
} from "../../../media/media-facts.js";
import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js";
import { resolveUserPath } from "../../../utils.js";
const URL_SCHEME_PATTERN = /^[a-z][a-z0-9+.-]*:/i;
const WINDOWS_DRIVE_PATH_PATTERN = /^[A-Za-z]:[\\/]/;
type DetectedImageRef = {
raw: string;
type: "path" | "media-uri";
resolved: string;
};
export type MediaImageRef = DetectedImageRef & {
aliases: string[];
detect?: boolean;
factIndex: number;
hydrate: boolean;
workspaceDir?: string;
};
export function isOpenClawCliImageCachePath(filePath: string): boolean {
const parts = filePath.replaceAll("\\", "/").split("/");
return parts.some((part, index) => {
if (part === ".openclaw-cli-images") {
return true;
}
const parent = parts[index - 1] ?? "";
return part === "openclaw-cli-images" && /^openclaw(?:-\d+)?$/.test(parent);
});
}
function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef | undefined {
if (!isImageMediaFact(fact)) {
return undefined;
}
const mediaUri = [fact.url, fact.path].find((value) => value?.startsWith("media://inbound/"));
const identity = mediaUri ?? fact.path ?? fact.url;
if (!identity) {
return fact.hydrationSuppressed === true
? {
aliases: [],
detect: false,
factIndex,
raw: "",
type: "path",
resolved: "",
hydrate: false,
...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}),
}
: undefined;
}
let resolved = mediaUri;
if (!resolved && identity && /^file:/i.test(identity)) {
try {
resolved = safeFileURLToPath(identity);
} catch {
resolved = undefined;
}
} else if (
!resolved &&
identity &&
(!URL_SCHEME_PATTERN.test(identity) || WINDOWS_DRIVE_PATH_PATTERN.test(identity))
) {
resolved = identity;
}
if (resolved?.startsWith("~")) {
resolved = resolveUserPath(resolved);
}
const hydrate = fact.hydrationSuppressed !== true;
if (!resolved || isOpenClawCliImageCachePath(resolved)) {
return {
aliases: [fact.path, fact.url].filter((value): value is string => Boolean(value)),
detect: false,
factIndex,
raw: identity,
type: "path",
resolved: identity,
hydrate: false,
...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}),
};
}
return {
aliases: [fact.path, fact.url, resolved].filter((value): value is string => Boolean(value)),
factIndex,
raw: mediaUri ?? fact.path ?? fact.url ?? resolved,
type: mediaUri ? "media-uri" : "path",
resolved,
hydrate,
...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}),
};
}
export function collectMediaImageRefs(
media?: readonly MediaFact[],
): Array<MediaImageRef | undefined> {
return normalizeMediaFacts(media).flatMap((fact, factIndex) =>
isImageMediaFact(fact) ? [mediaFactToImageRef(fact, factIndex)] : [],
);
}
export function collectIdentitylessMediaImageFactIndexes(media?: readonly MediaFact[]): number[] {
return normalizeMediaFacts(media).flatMap((fact, factIndex) =>
isImageMediaFact(fact) &&
fact.hydrationSuppressed !== true &&
fact.path === undefined &&
fact.url === undefined
? [factIndex]
: [],
);
}
// Guards for transports that cannot carry attachments (paired-node CLI): only
// facts that will actually hydrate an image count; described/remote-only facts
// whose hydration is suppressed must not block text-only prompts.
export function hasHydratableMediaImages(media?: readonly MediaFact[]): boolean {
return collectMediaImageRefs(media).some((ref) => ref?.hydrate === true);
}
export function selectMediaImageRefs(params: {
refs: Array<MediaImageRef | undefined>;
existingImageCount: number;
imageOrder?: readonly PromptImageOrderEntry[];
}): Array<MediaImageRef | undefined> {
const { refs } = params;
if (!params.imageOrder?.length) {
// Legacy turns (no layout metadata): identity-less facts are the inline
// images' own slots — pair them positionally so they cannot count as failed
// offloads; identity-bearing refs remain genuine offloaded attachments.
let inlinePairs = params.existingImageCount;
return refs.filter((ref) => {
if (ref === undefined && inlinePairs > 0) {
inlinePairs -= 1;
return false;
}
return true;
});
}
if (refs.length !== params.imageOrder.length) {
// Partial fact arrays cannot prove positional ownership. Keep every ref as
// an offload so no attachment is silently consumed by an inline slot.
return refs;
}
let remainingExisting = params.existingImageCount;
return params.imageOrder.flatMap((entry, index) => {
if (entry === "offloaded") {
return [refs[index]];
}
if (remainingExisting > 0) {
remainingExisting -= 1;
return [];
}
return [undefined];
});
}
@@ -0,0 +1,680 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import { createSolidPngBuffer } from "../../../../test/helpers/image-fixtures.js";
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
import {
attachRuntimePromptMediaFacts,
readRuntimePromptImageOrder,
readRuntimePromptMediaFacts,
} from "../../../media/media-facts.js";
import {
finalizeRuntimePromptImages,
readRuntimePromptImageFactIndexes,
} from "../../../media/runtime-prompt-image-provenance.js";
import { buildPersistedUserTurnMessage } from "../../../sessions/user-turn-transcript.js";
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
import type { AgentMessage } from "../../runtime/index.js";
import {
detectAndLoadPromptImages,
detectImageReferences,
hydratePromptMediaMessages,
} from "./images.js";
const TINY_PNG_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
describe("structured prompt media replay", () => {
it("keeps per-slot provenance when the same image object is reused", () => {
const sharedImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const { images } = finalizeRuntimePromptImages([
{ image: sharedImage, factIndex: 0 },
{ image: sharedImage, factIndex: 1 },
]);
expect(readRuntimePromptImageFactIndexes(images)).toEqual([0, 1]);
});
it("retains the runtime fact carrier when queued hydration fails", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-runtime-failure-"));
const media = [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }];
const message = attachRuntimePromptMediaFacts(
{ role: "user" as const, content: "missing attachment" },
media,
["offloaded"],
) as unknown as AgentMessage;
const runtimeMedia = readRuntimePromptMediaFacts(message);
try {
const result = await hydratePromptMediaMessages([message], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect(readRuntimePromptMediaFacts(result[0] as AgentMessage)).toEqual(runtimeMedia);
expect(readRuntimePromptImageOrder(result[0] as AgentMessage)).toEqual(["offloaded"]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("does not fail a described remote-only fact with no local identity", async () => {
const media = buildInboundMediaNoteProjection({
MediaPaths: [""],
MediaUrls: ["https://example.com/described.png"],
MediaTypes: ["image/png"],
MediaUnderstanding: [
{
kind: "image.description",
attachmentIndex: 0,
text: "already described",
provider: "test",
},
],
}).media;
const result = await detectAndLoadPromptImages({
prompt: "already described",
media,
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
});
expect(result.failedMediaCount).toBe(0);
expect(result.detectedRefs).toEqual([]);
expect(result.images).toEqual([]);
});
it("reports a fact-owned image dropped during sanitization", async () => {
const result = await detectAndLoadPromptImages({
prompt: "inspect it",
media: [{ path: "/tmp/already-materialized.png", contentType: "image/png" }],
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }],
existingImageFactIndexes: [0],
imageOrder: ["inline"],
});
expect(result.failedMediaCount).toBe(1);
expect(result.images).toEqual([]);
});
it("preserves persisted facts when replay hydration fails", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-failure-"));
const missingPath = path.join(workspaceDir, "missing.png");
const message = {
role: "user" as const,
content: "missing attachment",
MediaPath: missingPath,
MediaPaths: [missingPath],
MediaType: "image/png",
MediaTypes: ["image/png"],
} as unknown as AgentMessage;
try {
const result = await hydratePromptMediaMessages([message], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
const replayed = result[0] as unknown as Record<string, unknown>;
expect(replayed.MediaPaths).toEqual([missingPath]);
expect(replayed.content).toEqual([{ type: "text", text: "missing attachment" }]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("detects bracketed explicit paths but not textual claim-check tickets", () => {
expect(detectImageReferences("inspect [source /tmp/photo.png]")).toEqual([
{ raw: "/tmp/photo.png", type: "path", resolved: "/tmp/photo.png" },
]);
expect(detectImageReferences("[media attached: media://inbound/legacy.png]")).toEqual([]);
expect(detectImageReferences("[media attached: /tmp/legacy.png (image/png)]")).toEqual([]);
expect(detectImageReferences("[Image: source: /tmp/legacy.png]")).toEqual([]);
});
it("dedupes the rendered alias of a private macOS path", async () => {
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
try {
const result = await detectAndLoadPromptImages({
prompt: "[media attached: /var/tmp/aliased.png (image/png)]",
media: [{ path: "/private/var/tmp/aliased.png", contentType: "image/png" }],
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
});
expect(result.detectedRefs).toEqual([
{
raw: "/private/var/tmp/aliased.png",
resolved: "/private/var/tmp/aliased.png",
type: "path",
},
]);
expect(result.skippedCount).toBe(1);
} finally {
platformSpy.mockRestore();
}
});
it("keeps private-var and var paths distinct away from Darwin", () => {
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("linux");
try {
expect(
detectImageReferences("compare /private/var/tmp/a.png and /var/tmp/a.png"),
).toHaveLength(2);
} finally {
platformSpy.mockRestore();
}
});
it("expands a home-relative path carried by a media fact", async () => {
const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-fact-home-"));
const imagePath = path.join(homeDir, "Pictures", "photo.png");
await fs.mkdir(path.dirname(imagePath), { recursive: true });
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const envSnapshot = captureEnv(["HOME"]);
setTestEnvValue("HOME", homeDir);
try {
const result = await detectAndLoadPromptImages({
prompt: "describe it",
media: [{ path: "~/Pictures/photo.png", contentType: "image/png" }],
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
localRoots: [homeDir],
});
expect(result.images).toHaveLength(1);
} finally {
envSnapshot.restore();
await fs.rm(homeDir, { recursive: true, force: true });
}
});
it("keeps same-spelled relative refs from different workspaces distinct", async () => {
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relative-roots-"));
const stagedDir = path.join(rootDir, "staged");
const currentDir = path.join(rootDir, "current");
await fs.mkdir(stagedDir, { recursive: true });
await fs.mkdir(currentDir, { recursive: true });
await fs.writeFile(path.join(stagedDir, "photo.png"), Buffer.from(TINY_PNG_BASE64, "base64"));
await fs.writeFile(
path.join(currentDir, "photo.png"),
createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 }),
);
try {
const result = await detectAndLoadPromptImages({
prompt: "compare ./photo.png",
media: [{ path: "./photo.png", contentType: "image/png", workspaceDir: stagedDir }],
workspaceDir: currentDir,
model: { input: ["text", "image"] },
localRoots: [stagedDir, currentDir],
});
expect(result.loadedCount).toBe(2);
expect(result.images).toHaveLength(2);
} finally {
await fs.rm(rootDir, { recursive: true, force: true });
}
});
it("does not duplicate an already-materialized offloaded slot", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-materialized-offload-"));
const imagePath = path.join(workspaceDir, "offloaded.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const image = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const persisted = buildPersistedUserTurnMessage({
text: "already materialized",
media: [{ path: imagePath, contentType: "image/png" }],
mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] },
}) as unknown as AgentMessage;
try {
const first = await hydratePromptMediaMessages([persisted], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
const serialized = JSON.stringify(first[0]);
const restored = JSON.parse(serialized) as AgentMessage;
await fs.rm(imagePath);
const replay = await hydratePromptMediaMessages([restored], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((replay[0] as unknown as { content?: unknown }).content).toEqual([
{ type: "text", text: "already materialized" },
image,
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("prefers persisted fact-index layout when runtime order is also present", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-layout-authority-"));
const describedPath = path.join(workspaceDir, "described.png");
const inlinePath = path.join(workspaceDir, "inline.png");
const offloadedPath = path.join(workspaceDir, "offloaded.png");
const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 });
await fs.writeFile(describedPath, createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 }));
await fs.writeFile(inlinePath, Buffer.from(TINY_PNG_BASE64, "base64"));
await fs.writeFile(offloadedPath, offloadedBuffer);
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const media = [
{ path: describedPath, contentType: "image/png" },
{ path: inlinePath, contentType: "image/png" },
{ path: offloadedPath, contentType: "image/png" },
];
const persisted = buildPersistedUserTurnMessage({
text: "compare",
media,
mediaImageLayout: {
slots: [
{ kind: "inline", factIndex: 1 },
{ kind: "offloaded", factIndex: 2 },
],
suppressedFactIndexes: [0],
},
});
const message = attachRuntimePromptMediaFacts(
{
...persisted,
content: [{ type: "text" as const, text: "compare" }, inlineImage],
} as AgentMessage,
media,
["inline", "offloaded"],
);
try {
const first = await hydratePromptMediaMessages([message], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
const firstContent = (first[0] as unknown as { content?: unknown[] }).content;
expect(firstContent).toEqual([
{ type: "text", text: "compare" },
inlineImage,
{ type: "image", data: offloadedBuffer.toString("base64"), mimeType: "image/png" },
]);
const serialized = JSON.stringify(first[0]);
await fs.rm(describedPath);
await fs.rm(inlinePath);
await fs.rm(offloadedPath);
const replay = await hydratePromptMediaMessages([JSON.parse(serialized) as AgentMessage], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((replay[0] as unknown as { content?: unknown[] }).content).toEqual(firstContent);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("retries an offloaded fact even when an unrelated explicit image exists", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-offload-retry-"));
const explicitImage = {
type: "image" as const,
data: TINY_PNG_BASE64,
mimeType: "image/png",
};
try {
const result = await detectAndLoadPromptImages({
prompt: "retry missing attachment",
media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }],
mediaImageLayout: {
slots: [{ kind: "offloaded", factIndex: 0 }],
suppressedFactIndexes: [],
},
workspaceDir,
model: { input: ["text", "image"] },
existingImages: [explicitImage],
existingImageFactIndexes: [null],
workspaceOnly: true,
});
expect(result.failedMediaCount).toBe(1);
expect(result.images).toEqual([explicitImage]);
expect(result.imageFactIndexes).toEqual([null]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("does not assign a partial offloaded fact to an existing inline image", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-facts-"));
const offloadedPath = path.join(workspaceDir, "offloaded.png");
const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 });
await fs.writeFile(offloadedPath, offloadedBuffer);
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
try {
const result = await detectAndLoadPromptImages({
prompt: "compare",
media: [{ path: offloadedPath, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
existingImages: [inlineImage],
imageOrder: ["inline", "offloaded"],
workspaceOnly: true,
});
expect(result.failedMediaCount).toBe(0);
expect(result.loadedCount).toBe(1);
expect(result.images).toEqual([
inlineImage,
{ type: "image", data: offloadedBuffer.toString("base64"), mimeType: "image/png" },
]);
expect(result.imageFactIndexes).toEqual([null, 0]);
expect(readRuntimePromptImageFactIndexes(result.images)).toEqual([null, 0]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("preserves fact order for partial hydration without persisted layout", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-order-"));
const firstPath = path.join(workspaceDir, "first.png");
const secondPath = path.join(workspaceDir, "second.png");
const thirdPath = path.join(workspaceDir, "third.png");
const secondBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 });
const thirdBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 });
await fs.writeFile(thirdPath, thirdBuffer);
const firstImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const secondImage = {
type: "image" as const,
data: secondBuffer.toString("base64"),
mimeType: "image/png",
};
try {
const result = await detectAndLoadPromptImages({
prompt: "compare",
media: [
{ path: firstPath, contentType: "image/png" },
{ path: secondPath, contentType: "image/png" },
{ path: thirdPath, contentType: "image/png" },
],
workspaceDir,
model: { input: ["text", "image"] },
existingImages: [firstImage, secondImage],
existingImageFactIndexes: [0, 1],
imageOrder: ["offloaded", "inline", "offloaded"],
workspaceOnly: true,
});
expect(result.failedMediaCount).toBe(0);
expect(result.images).toEqual([
firstImage,
secondImage,
{ type: "image", data: thirdBuffer.toString("base64"), mimeType: "image/png" },
]);
expect(result.imageFactIndexes).toEqual([0, 1, 2]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("uses an unowned block for an inline slot when exact provenance is unavailable", async () => {
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const offloadedImage = {
type: "image" as const,
data: createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 }).toString("base64"),
mimeType: "image/png",
};
const result = await detectAndLoadPromptImages({
prompt: "compare",
media: [{ kind: "document" }, { kind: "image" }, { kind: "image" }],
mediaImageLayout: {
slots: [
{ kind: "inline", factIndex: 1 },
{ kind: "offloaded", factIndex: 2 },
],
suppressedFactIndexes: [],
},
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
existingImages: [inlineImage, offloadedImage],
existingImageFactIndexes: [null, 2],
});
expect(result.failedMediaCount).toBe(0);
expect(result.images).toEqual([inlineImage, offloadedImage]);
expect(result.imageFactIndexes).toEqual([null, 2]);
});
it("uses explicit inline layout ownership for sanitization failures", async () => {
const result = await detectAndLoadPromptImages({
prompt: "inspect",
media: [{ kind: "image" }, { kind: "image" }],
mediaImageLayout: {
slots: [{ kind: "inline", factIndex: 1 }],
suppressedFactIndexes: [0],
},
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }],
});
expect(result.failedMediaCount).toBe(1);
expect(result.images).toEqual([]);
});
it("reports a missing unsuppressed inline layout slot", async () => {
const result = await detectAndLoadPromptImages({
prompt: "inspect",
media: [{ kind: "image" }],
mediaImageLayout: {
slots: [{ kind: "inline", factIndex: 0 }],
suppressedFactIndexes: [],
},
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
});
expect(result.failedMediaCount).toBe(1);
expect(result.images).toEqual([]);
});
it("pairs an identity-less legacy fact with its existing inline block", async () => {
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const message = {
role: "user" as const,
content: [{ type: "text" as const, text: "legacy identity-less" }, inlineImage],
media: [{ kind: "image" }],
} as unknown as AgentMessage;
const result = await hydratePromptMediaMessages([message], {
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
});
expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([
{ type: "text", text: "legacy identity-less" },
inlineImage,
]);
});
it("does not pair an unresolved remote legacy fact with an existing inline block", async () => {
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const message = {
role: "user" as const,
content: [{ type: "text" as const, text: "remote identity" }, inlineImage],
media: [{ kind: "image", url: "https://example.test/remote.png" }],
} as unknown as AgentMessage;
const result = await hydratePromptMediaMessages([message], {
workspaceDir: "/tmp",
model: { input: ["text", "image"] },
});
const meta = (result[0] as unknown as Record<string, unknown>)["__openclaw"] as
| Record<string, unknown>
| undefined;
expect(meta?.mediaImageBlockFactIndexes).toEqual([null]);
expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([
{ type: "text", text: "remote identity" },
inlineImage,
]);
});
it("hydrates an identity-bearing legacy fact beside an unowned inline block", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-legacy-inline-"));
const imagePath = path.join(workspaceDir, "inline.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const message = {
role: "user" as const,
content: [{ type: "text" as const, text: "legacy" }, inlineImage],
MediaPath: imagePath,
MediaPaths: [imagePath],
MediaType: "image/png",
MediaTypes: ["image/png"],
} as unknown as AgentMessage;
try {
const result = await hydratePromptMediaMessages([message], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
{ type: "text", text: "legacy" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
inlineImage,
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("keeps identity-bearing legacy facts ahead of an unowned inline block", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-legacy-order-"));
const existingPath = path.join(workspaceDir, "existing.png");
const hydratedPath = path.join(workspaceDir, "hydrated.png");
const hydratedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 });
await fs.writeFile(existingPath, Buffer.from(TINY_PNG_BASE64, "base64"));
await fs.writeFile(hydratedPath, hydratedBuffer);
const existingImage = {
type: "image" as const,
data: TINY_PNG_BASE64,
mimeType: "image/png",
};
const message = {
role: "user" as const,
content: [{ type: "text" as const, text: "legacy order" }, existingImage],
MediaPath: existingPath,
MediaPaths: [existingPath, hydratedPath],
MediaType: "image/png",
MediaTypes: ["image/png", "image/png"],
} as unknown as AgentMessage;
try {
const result = await hydratePromptMediaMessages([message], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([
{ type: "text", text: "legacy order" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
{ type: "image", data: hydratedBuffer.toString("base64"), mimeType: "image/png" },
existingImage,
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("dedupes the local path alias of a claim-check fact", async () => {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-alias-"));
const workspaceDir = path.join(stateDir, "workspace");
const inboundDir = path.join(stateDir, "media", "inbound");
const mediaId = "aliased.png";
const imagePath = path.join(inboundDir, mediaId);
await fs.mkdir(workspaceDir, { recursive: true });
await fs.mkdir(inboundDir, { recursive: true });
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
try {
const result = await detectAndLoadPromptImages({
prompt: `[media attached: ${imagePath} (image/png)]`,
media: [{ path: imagePath, url: `media://inbound/${mediaId}`, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
});
expect(result.loadedCount).toBe(1);
expect(result.images).toHaveLength(1);
expect(result.detectedRefs).toEqual([
{
raw: `media://inbound/${mediaId}`,
resolved: `media://inbound/${mediaId}`,
type: "media-uri",
},
]);
} finally {
envSnapshot.restore();
await fs.rm(stateDir, { recursive: true, force: true });
}
});
it("keeps described image facts suppressed across serialize and restore", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-suppressed-"));
const imagePath = path.join(workspaceDir, "described.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const serialized = JSON.stringify(
buildPersistedUserTurnMessage({
text: "description already present",
media: [
{ path: imagePath, contentType: "image/png", hydrationSuppressed: true },
{ kind: "sticker", hydrationSuppressed: true },
],
}),
);
const restored = JSON.parse(serialized) as AgentMessage;
const meta = (restored as unknown as Record<string, unknown>)["__openclaw"] as
| Record<string, unknown>
| undefined;
const persistedMedia = meta?.media as
| Array<{ path?: string; contentType?: string; hydrationSuppressed?: boolean }>
| undefined;
try {
expect(persistedMedia).toEqual([
expect.objectContaining({
path: imagePath,
contentType: "image/png",
hydrationSuppressed: true,
}),
expect.objectContaining({ kind: "sticker", hydrationSuppressed: true }),
]);
const replayHydration = await detectAndLoadPromptImages({
prompt: "description already present",
media: persistedMedia,
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect(replayHydration.failedMediaCount).toBe(0);
expect(replayHydration.images).toEqual([]);
const result = await hydratePromptMediaMessages([restored], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
{ type: "text", text: "description already present" },
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
});
@@ -5,11 +5,24 @@ import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { describe, expect, it, vi } from "vitest";
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
import { resolvePreferredOpenClawTmpDir } from "../../../infra/tmp-openclaw-dir.js";
import {
attachRuntimePromptMediaFacts,
readRuntimePromptImageOrder,
} from "../../../media/media-facts.js";
import {
buildPersistedUserTurnMessage,
mergePreparedUserTurnMessageForRuntime,
} from "../../../sessions/user-turn-transcript.js";
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js";
import type { AgentMessage } from "../../runtime/index.js";
import { createUnsafeMountedSandbox } from "../../test-helpers/unsafe-mounted-sandbox.js";
import { detectAndLoadPromptImages, detectImageReferences, loadImageFromRef } from "./images.js";
import {
detectAndLoadPromptImages,
detectImageReferences,
hydratePromptMediaMessages,
} from "./images.js";
const TINY_PNG_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
@@ -154,10 +167,10 @@ describe("detectImageReferences", () => {
});
it("does not leak parser state between calls", () => {
expect(detectImageReferences("[media attached: /tmp/first.png (image/png)]")).toStrictEqual([
expect(detectImageReferences("See /tmp/first.png")).toStrictEqual([
{ raw: "/tmp/first.png", type: "path", resolved: "/tmp/first.png" },
]);
expect(detectImageReferences("[Image: source: /tmp/second.jpg]")).toStrictEqual([
expect(detectImageReferences("See /tmp/second.jpg")).toStrictEqual([
{ raw: "/tmp/second.jpg", type: "path", resolved: "/tmp/second.jpg" },
]);
const thirdPath = path.join(os.tmpdir(), "third.webp");
@@ -265,67 +278,6 @@ describe("detectImageReferences", () => {
});
});
it("detects [Image: source: ...] format from messaging systems", () => {
const ref = expectSingleImageReference(`What does this image show?
[Image: source: /Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg]`);
expect(ref).toStrictEqual({
raw: "/Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg",
type: "path",
resolved: "/Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg",
});
});
it("handles complex message attachment paths", () => {
const ref = expectSingleImageReference(
"[Image: source: /Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg]",
);
expect(ref).toStrictEqual({
raw: "/Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg",
type: "path",
resolved:
"/Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg",
});
});
it("detects multiple images in [media attached: ...] format", () => {
// Multi-file format uses separate brackets on separate lines
const refs = expectImageReferenceCount(
`[media attached: 2 files]
[media attached 1/2: /Users/tyleryust/.openclaw/media/IMG_6430.jpeg (image/jpeg)]
[media attached 2/2: /Users/tyleryust/.openclaw/media/IMG_6431.jpeg (image/jpeg)]
what about these images?`,
2,
);
expect(refs).toStrictEqual([
{
raw: "/Users/tyleryust/.openclaw/media/IMG_6430.jpeg",
type: "path",
resolved: "/Users/tyleryust/.openclaw/media/IMG_6430.jpeg",
},
{
raw: "/Users/tyleryust/.openclaw/media/IMG_6431.jpeg",
type: "path",
resolved: "/Users/tyleryust/.openclaw/media/IMG_6431.jpeg",
},
]);
});
it("does not double-count path and url in same bracket", () => {
// Single file with URL (| separates path from url, not multiple files)
const ref = expectSingleImageReference(
"[media attached: /cache/IMG_6430.jpeg (image/jpeg) | /cache/IMG_6430.jpeg]",
);
expect(ref).toStrictEqual({
raw: "/cache/IMG_6430.jpeg",
type: "path",
resolved: "/cache/IMG_6430.jpeg",
});
});
it("ignores remote URLs entirely (local-only)", () => {
const refs = expectImageReferenceCount(
`To send an image: MEDIA:https://example.com/image.jpg
@@ -343,31 +295,6 @@ Also https://cdn.mysite.com/img.jpg`,
]);
});
it("handles single file format with URL (no index)", () => {
const ref =
expectSingleImageReference(`[media attached: /cache/photo.jpeg (image/jpeg) | https://example.com/url]
what is this?`);
expect(ref).toStrictEqual({
raw: "/cache/photo.jpeg",
type: "path",
resolved: "/cache/photo.jpeg",
});
});
it("handles paths with spaces in filename", () => {
// URL after | is https, not a local path, so only the local path should be detected
const ref =
expectSingleImageReference(`[media attached: /Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png (image/png) | https://example.com/same.png]
what is this?`);
expect(ref).toStrictEqual({
raw: "/Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png",
type: "path",
resolved: "/Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png",
});
});
it("ignores remote-host file URLs", () => {
expectNoImageReferences("See file://attacker/share/evil.png");
});
@@ -385,107 +312,6 @@ what is this?`);
});
});
describe("loadImageFromRef", () => {
it("hydrates managed inbound media URIs before workspace path resolution", async () => {
// Managed media URIs are canonical inbound attachment handles and should
// work even when workspaceOnly would reject ordinary outside paths.
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-uri-"));
const workspaceDir = path.join(stateDir, "workspace-agent");
const inboundDir = path.join(stateDir, "media", "inbound");
const mediaId = "telegram-photo.png";
await fs.mkdir(workspaceDir, { recursive: true });
await fs.mkdir(inboundDir, { recursive: true });
await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64"));
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
try {
const image = await loadImageFromRef(
{
raw: `media://inbound/${mediaId}`,
type: "media-uri",
resolved: `media://inbound/${mediaId}`,
},
workspaceDir,
{ workspaceOnly: true },
);
expect(image?.type).toBe("image");
expect(image?.mimeType).toBe("image/png");
expect(image?.data).toBe(TINY_PNG_BASE64);
} finally {
envSnapshot.restore();
await fs.rm(stateDir, { recursive: true, force: true });
}
});
it("hydrates sandbox-staged inbound media URIs", async () => {
const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-sbx-uri-"));
const inboundDir = path.join(sandboxRoot, "media", "inbound");
const mediaId = "telegram-photo.png";
await fs.mkdir(inboundDir, { recursive: true });
await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64"));
try {
const image = await loadImageFromRef(
{
raw: `media://inbound/${mediaId}`,
type: "media-uri",
resolved: `media://inbound/${mediaId}`,
},
sandboxRoot,
{
workspaceOnly: true,
sandbox: {
root: sandboxRoot,
bridge: createHostSandboxFsBridge(sandboxRoot),
},
},
);
expect(image?.type).toBe("image");
expect(image?.mimeType).toBe("image/png");
expect(image?.data).toBe(TINY_PNG_BASE64);
} finally {
await fs.rm(sandboxRoot, { recursive: true, force: true });
}
});
it("allows sandbox-validated host paths outside default media roots", async () => {
const homeDir = os.homedir();
await fs.mkdir(homeDir, { recursive: true });
const sandboxParent = await fs.mkdtemp(path.join(homeDir, "openclaw-sandbox-image-"));
try {
const sandboxRoot = path.join(sandboxParent, "sandbox");
await fs.mkdir(sandboxRoot, { recursive: true });
const imagePath = path.join(sandboxRoot, "photo.png");
const pngB64 = TINY_PNG_BASE64;
await fs.writeFile(imagePath, Buffer.from(pngB64, "base64"));
const image = await loadImageFromRef(
{
raw: "./photo.png",
type: "path",
resolved: "./photo.png",
},
sandboxRoot,
{
sandbox: {
root: sandboxRoot,
bridge: createHostSandboxFsBridge(sandboxRoot),
},
},
);
expect(image?.type).toBe("image");
expect(image?.mimeType).toBe("image/png");
expect(image?.data).toBe(TINY_PNG_BASE64);
} finally {
await fs.rm(sandboxParent, { recursive: true, force: true });
}
});
});
describe("detectAndLoadPromptImages", () => {
it("returns no images for non-vision models even when existing images are provided", async () => {
const result = await detectAndLoadPromptImages({
@@ -529,6 +355,7 @@ describe("detectAndLoadPromptImages", () => {
try {
const result = await detectAndLoadPromptImages({
prompt: "[media attached: ./photo.png (image/png)]\ndescribe it",
media: [{ path: imagePath, contentType: "image/png" }],
workspaceDir: stateDir,
model: { input: ["text", "image"] },
existingImages: [{ type: "image", data: pngB64, mimeType: "image/png" }],
@@ -536,7 +363,7 @@ describe("detectAndLoadPromptImages", () => {
workspaceOnly: true,
});
expect(result.detectedRefs).toHaveLength(1);
expect(result.detectedRefs).toEqual([{ raw: imagePath, type: "path", resolved: imagePath }]);
expect(result.loadedCount).toBe(0);
expect(result.skippedCount).toBe(0);
expect(result.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]);
@@ -545,6 +372,65 @@ describe("detectAndLoadPromptImages", () => {
}
});
it("uses a described fact identity to suppress its generated media-note path", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-described-dedupe-"));
const imagePath = path.join(workspaceDir, "photo.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
try {
const result = await detectAndLoadPromptImages({
prompt: `[media attached: ${imagePath} (image/png)]`,
media: buildInboundMediaNoteProjection({
MediaPath: imagePath,
MediaType: "image/png",
MediaUnderstanding: [
{
kind: "image.description",
attachmentIndex: 0,
text: "already described",
provider: "test",
},
],
}).media,
workspaceDir,
model: { input: ["text", "image"] },
});
expect(result.detectedRefs).toEqual([{ raw: imagePath, type: "path", resolved: imagePath }]);
expect(result.loadedCount).toBe(0);
expect(result.images).toEqual([]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("dedupes a relative fact projection against the fact workspace", async () => {
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-fact-workspace-dedupe-"));
const stagedDir = path.join(rootDir, "staged");
const currentDir = path.join(rootDir, "current");
await fs.mkdir(stagedDir, { recursive: true });
await fs.mkdir(currentDir, { recursive: true });
await fs.writeFile(path.join(stagedDir, "photo.png"), Buffer.from(TINY_PNG_BASE64, "base64"));
await fs.writeFile(path.join(currentDir, "photo.png"), TINY_GIF_BUFFER);
try {
const result = await detectAndLoadPromptImages({
prompt: "[media attached: ./photo.png (image/png)]",
media: [{ path: "./photo.png", contentType: "image/png", workspaceDir: stagedDir }],
workspaceDir: currentDir,
model: { input: ["text", "image"] },
localRoots: [stagedDir, currentDir],
});
expect(result.loadedCount).toBe(1);
expect(result.images).toEqual([
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
} finally {
await fs.rm(rootDir, { recursive: true, force: true });
}
});
it("keeps distinct inline attachments with identical bytes", async () => {
const pngB64 = TINY_PNG_BASE64;
const image = { type: "image" as const, data: pngB64, mimeType: "image/png" };
@@ -561,6 +447,37 @@ describe("detectAndLoadPromptImages", () => {
expect(result.images).toEqual([image, image]);
});
it("keeps offloaded-only facts when existing images have no order metadata", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-unordered-images-"));
const imagePath = path.join(workspaceDir, "offloaded.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const inlineImage = {
type: "image" as const,
data: TINY_GIF_BUFFER.toString("base64"),
mimeType: "image/gif",
};
try {
const result = await detectAndLoadPromptImages({
prompt: "compare",
media: [{ path: imagePath, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
existingImages: [inlineImage],
imageOrder: [],
workspaceOnly: true,
});
expect(result.loadedCount).toBe(1);
expect(result.images).toEqual([
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
inlineImage,
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("classifies prompt and attachment refs while preserving mixed attachment order", async () => {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-order-"));
const inboundDir = path.join(stateDir, "media", "inbound");
@@ -579,6 +496,7 @@ describe("detectAndLoadPromptImages", () => {
try {
const result = await detectAndLoadPromptImages({
prompt,
media: [{ url: "media://inbound/att-b.gif", contentType: "image/gif" }],
workspaceDir: stateDir,
model: { input: ["text", "image"] },
existingImages: [{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }],
@@ -587,11 +505,6 @@ describe("detectAndLoadPromptImages", () => {
});
expect(result.detectedRefs).toEqual([
{
raw: "media://inbound/prompt-ref.png",
type: "media-uri",
resolved: "media://inbound/prompt-ref.png",
},
{
raw: "media://inbound/att-b.gif",
type: "media-uri",
@@ -599,12 +512,11 @@ describe("detectAndLoadPromptImages", () => {
},
{ raw: "./prompt-b.png", type: "path", resolved: "./prompt-b.png" },
]);
expect(result.loadedCount).toBe(3);
expect(result.loadedCount).toBe(2);
expect(result.images).toEqual([
{ type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
} finally {
envSnapshot.restore();
@@ -612,6 +524,63 @@ describe("detectAndLoadPromptImages", () => {
}
});
it("preserves an empty described-image slot in mixed attachment order", async () => {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-described-image-order-"));
const inboundDir = path.join(stateDir, "media", "inbound");
const mediaId = "remaining.gif";
await fs.mkdir(inboundDir, { recursive: true });
await fs.writeFile(path.join(inboundDir, mediaId), TINY_GIF_BUFFER);
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
try {
const result = await detectAndLoadPromptImages({
prompt: "compare",
media: [
{ kind: "image" },
{ kind: "image" },
{ url: `media://inbound/${mediaId}`, contentType: "image/gif" },
],
workspaceDir: stateDir,
model: { input: ["text", "image"] },
existingImages: [inlineImage],
imageOrder: ["offloaded", "inline", "offloaded"],
});
expect(result.images).toEqual([
inlineImage,
{ type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" },
]);
} finally {
envSnapshot.restore();
await fs.rm(stateDir, { recursive: true, force: true });
}
});
it("does not load an explicit prompt ref twice when the same fact owns it", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-fact-"));
await fs.writeFile(path.join(workspaceDir, "same.png"), Buffer.from(TINY_PNG_BASE64, "base64"));
try {
const result = await detectAndLoadPromptImages({
prompt: "Compare ./same.png",
media: [{ path: "./same.png", contentType: "image/png", workspaceDir }],
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect(result.detectedRefs).toEqual([
{ raw: "./same.png", type: "path", resolved: "./same.png" },
]);
expect(result.loadedCount).toBe(1);
expect(result.images).toHaveLength(1);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("blocks prompt image refs outside workspace when sandbox workspaceOnly is enabled", async () => {
// Sandbox workspaceOnly uses the bridge to validate mounted paths; ordinary
// prompt refs outside the workspace are detected but intentionally skipped.
@@ -675,4 +644,220 @@ describe("detectAndLoadPromptImages", () => {
await fs.rm(stateDir, { recursive: true, force: true });
}
});
it("keeps the fact hydration size limit", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-size-"));
const imagePath = path.join(workspaceDir, "too-large.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
try {
const result = await detectAndLoadPromptImages({
prompt: "describe it",
media: [{ path: imagePath, contentType: "image/png" }],
workspaceDir,
model: { input: ["text", "image"] },
maxBytes: 1,
});
expect(result.loadedCount).toBe(0);
expect(result.failedMediaCount).toBe(1);
expect(result.skippedCount).toBe(1);
expect(result.images).toHaveLength(0);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
});
describe("hydratePromptMediaMessages", () => {
it("hydrates queued facts in attachment order without mutating cache-stable input", async () => {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-queued-image-facts-"));
const inboundDir = path.join(stateDir, "media", "inbound");
const mediaId = "queued.gif";
await fs.mkdir(inboundDir, { recursive: true });
await fs.writeFile(path.join(inboundDir, mediaId), TINY_GIF_BUFFER);
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const message = attachRuntimePromptMediaFacts(
{ role: "user" as const, content: [{ type: "text" as const, text: "compare" }, inlineImage] },
[{ url: `media://inbound/${mediaId}`, contentType: "image/gif" }],
["offloaded", "inline"],
);
const options = {
workspaceDir: stateDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
};
try {
const runtimeMessage = message as unknown as AgentMessage;
const first = await hydratePromptMediaMessages([runtimeMessage], options);
const second = await hydratePromptMediaMessages([runtimeMessage], options);
const firstContent = (first?.[0] as unknown as { content?: unknown })?.content;
const secondContent = (second?.[0] as unknown as { content?: unknown })?.content;
expect(firstContent).toEqual([
{ type: "text", text: "compare" },
{ type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" },
inlineImage,
]);
expect(secondContent).toEqual(firstContent);
expect(message.content).toEqual([{ type: "text", text: "compare" }, inlineImage]);
} finally {
envSnapshot.restore();
await fs.rm(stateDir, { recursive: true, force: true });
}
});
it("hydrates a fact-owned user message whose content is a string", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-string-image-facts-"));
const imagePath = path.join(workspaceDir, "photo.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const message = attachRuntimePromptMediaFacts(
{ role: "user" as const, content: "describe it" },
[{ path: imagePath, contentType: "image/png" }],
) as unknown as AgentMessage;
try {
const result = await hydratePromptMediaMessages([message], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
{ type: "text", text: "describe it" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("reconstructs recent facts from serialized transcript media fields", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replayed-image-"));
const imagePath = path.join(workspaceDir, "photo.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const message = {
role: "user" as const,
content: "describe the replayed image",
MediaPath: imagePath,
MediaPaths: [imagePath],
MediaType: "image/png",
MediaTypes: ["image/png"],
} as unknown as AgentMessage;
try {
const result = await hydratePromptMediaMessages([message], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
{ type: "text", text: "describe the replayed image" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("preserves offloaded-before-inline order across serialize and restore", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-order-"));
const offloadedPath = path.join(workspaceDir, "offloaded.png");
const inlinePath = path.join(workspaceDir, "inline.gif");
await fs.writeFile(offloadedPath, Buffer.from(TINY_PNG_BASE64, "base64"));
await fs.writeFile(inlinePath, TINY_GIF_BUFFER);
const inlineImage = {
type: "image" as const,
data: TINY_GIF_BUFFER.toString("base64"),
mimeType: "image/gif",
};
const runtime = attachRuntimePromptMediaFacts(
{ role: "user" as const, content: [{ type: "text" as const, text: "compare" }, inlineImage] },
[{ path: offloadedPath, contentType: "image/png" }],
["offloaded", "inline"],
) as unknown as AgentMessage;
const persisted = buildPersistedUserTurnMessage({
text: "compare",
media: [
{ path: offloadedPath, contentType: "image/png" },
{ path: inlinePath, contentType: "image/gif" },
],
mediaImageLayout: {
slots: [
{ kind: "offloaded", factIndex: 0 },
{ kind: "inline", factIndex: 1 },
],
},
});
const serialized = JSON.stringify(
mergePreparedUserTurnMessageForRuntime({
runtimeMessage: runtime,
preparedMessage: persisted,
}),
);
const restored = JSON.parse(serialized) as AgentMessage;
try {
expect(readRuntimePromptImageOrder(restored)).toBeUndefined();
const result = await hydratePromptMediaMessages([restored], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
{ type: "text", text: "compare" },
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
inlineImage,
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("keeps duplicate fact slots across serialize and restore", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-duplicates-"));
const imagePath = path.join(workspaceDir, "same.png");
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const persisted = buildPersistedUserTurnMessage({
text: "compare duplicates",
media: [
{ path: imagePath, contentType: "image/png" },
{ path: imagePath, contentType: "image/png" },
],
mediaImageLayout: {
slots: [
{ kind: "offloaded", factIndex: 0 },
{ kind: "inline", factIndex: 1 },
],
},
});
const serialized = JSON.stringify(
mergePreparedUserTurnMessageForRuntime({
runtimeMessage: {
role: "user",
content: [{ type: "text", text: "compare duplicates" }, inlineImage],
} as AgentMessage,
preparedMessage: persisted,
}),
);
const restored = JSON.parse(serialized) as AgentMessage;
try {
const result = await hydratePromptMediaMessages([restored], {
workspaceDir,
model: { input: ["text", "image"] },
workspaceOnly: true,
});
const content = (result[0] as unknown as { content?: unknown[] }).content ?? [];
expect(content.filter((block) => (block as { type?: unknown }).type === "image")).toEqual([
inlineImage,
inlineImage,
]);
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,113 @@
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
import { isImageMediaFact, resolveMediaFacts } from "../../../media/media-facts.js";
import { resolveImageSanitizationLimits } from "../../image-sanitization.js";
import { resolveAttemptWorkspaceSandbox } from "./attempt-setup.js";
import { detectAndLoadPromptImages } from "./images.js";
import type { RunEmbeddedAgentParams } from "./params.js";
import {
readPersistedMediaImageLayout,
readPersistedPromptMediaFacts,
} from "./prompt-image-metadata.js";
import type { EmbeddedRunAttemptParams } from "./types.js";
function toTypeOnlyImageFact(
fact: NonNullable<RunEmbeddedAgentParams["media"]>[number],
hydrationSuppressed: boolean,
): NonNullable<RunEmbeddedAgentParams["media"]>[number] {
return {
contentType: fact.contentType,
kind: fact.kind === "sticker" ? "sticker" : "image",
messageId: fact.messageId,
transcribed: fact.transcribed,
...(fact.hydrationSuppressed === true || hydrationSuppressed
? { hydrationSuppressed: true }
: {}),
};
}
/** Materializes fact-carried images before a plugin harness owns transport. */
export async function preparePluginHarnessPromptImages(params: {
runParams: RunEmbeddedAgentParams;
runtime: {
sessionId: string;
sessionKey?: string;
workspaceDir: string;
model: EmbeddedRunAttemptParams["model"];
};
pluginHarnessOwnsTransport: boolean;
}): Promise<{
images: RunEmbeddedAgentParams["images"];
imageOrder: RunEmbeddedAgentParams["imageOrder"];
media: RunEmbeddedAgentParams["media"];
}> {
const { runParams, runtime } = params;
if (!params.pluginHarnessOwnsTransport) {
return {
images: runParams.images,
imageOrder: runParams.imageOrder,
media: runParams.media,
};
}
const persistedMessage =
runParams.userTurnTranscriptRecorder?.message ??
(await runParams.userTurnTranscriptRecorder?.resolveMessage());
const persistedMedia = persistedMessage
? (readPersistedPromptMediaFacts(persistedMessage) ??
resolveMediaFacts(persistedMessage as unknown as Parameters<typeof resolveMediaFacts>[0]))
: [];
const hydrationMedia = persistedMedia.length > 0 ? persistedMedia : runParams.media;
if (!hydrationMedia?.some(isImageMediaFact)) {
return {
images: runParams.images,
imageOrder: runParams.imageOrder,
media: runParams.media,
};
}
const workspace = await resolveAttemptWorkspaceSandbox({
...runParams,
cwd: undefined,
sessionId: runtime.sessionId,
sessionKey: runtime.sessionKey,
workspaceDir: runtime.workspaceDir,
});
const result = await detectAndLoadPromptImages({
prompt: "",
media: hydrationMedia,
mediaImageLayout: persistedMessage
? readPersistedMediaImageLayout(persistedMessage)
: undefined,
workspaceDir: workspace.effectiveWorkspace,
model: runtime.model,
existingImages: runParams.images,
imageOrder: runParams.imageOrder,
maxBytes: MAX_IMAGE_BYTES,
maxDimensionPx: resolveImageSanitizationLimits(runParams.config).maxDimensionPx,
localRoots: workspace.effectiveFsWorkspaceOnly
? [workspace.effectiveWorkspace, workspace.resolvedWorkspace]
: undefined,
workspaceOnly: workspace.effectiveFsWorkspaceOnly,
sandbox:
workspace.sandbox?.enabled && workspace.sandbox.fsBridge
? { root: workspace.sandbox.workspaceDir, bridge: workspace.sandbox.fsBridge }
: undefined,
});
if (result.failedMediaCount > 0) {
throw new Error(
`failed to hydrate ${result.failedMediaCount} structured image attachment(s) for plugin harness input`,
);
}
const materializedFactIndexes = new Set(
result.imageFactIndexes.filter((index): index is number => index !== null),
);
const retainedMedia = hydrationMedia?.map((fact, factIndex) =>
isImageMediaFact(fact)
? toTypeOnlyImageFact(fact, !materializedFactIndexes.has(factIndex))
: fact,
);
return {
images: result.images,
imageOrder: result.images.length > 0 ? result.images.map(() => "inline" as const) : undefined,
media: retainedMedia?.length ? retainedMedia : undefined,
};
}
@@ -0,0 +1,118 @@
import { normalizeMediaFacts, type MediaFact } from "../../../media/media-facts.js";
import type { AgentMessage } from "../../runtime/index.js";
export type ImageFactIndex = number | null;
export type MediaImageLayout = {
slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>;
suppressedFactIndexes: number[];
};
export function resolveLayoutInlineFactIndexes(
layout: MediaImageLayout | undefined,
existingImageCount: number,
): ImageFactIndex[] | undefined {
const factIndexes = layout?.slots.flatMap((slot) =>
slot.kind === "inline" ? [slot.factIndex ?? null] : [],
);
return factIndexes?.length === existingImageCount ? factIndexes : undefined;
}
export function countMissingLayoutInlineSlots(
layout: MediaImageLayout | undefined,
existingFactIndexes: readonly ImageFactIndex[] | undefined,
existingImageCount: number,
): number {
if (!layout) {
return 0;
}
const available = existingFactIndexes
? [...existingFactIndexes]
: Array.from({ length: existingImageCount }, () => null);
let missing = 0;
for (const slot of layout.slots) {
if (slot.kind !== "inline") {
continue;
}
const exactIndex =
slot.factIndex === undefined
? available.length > 0
? 0
: -1
: available.findIndex((factIndex) => factIndex === slot.factIndex);
const matchIndex = exactIndex >= 0 ? exactIndex : available.indexOf(null);
if (matchIndex >= 0) {
available.splice(matchIndex, 1);
} else {
missing++;
}
}
return missing;
}
export function readPersistedImageBlockFactIndexes(
message: AgentMessage,
): ImageFactIndex[] | undefined {
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
const value =
meta && typeof meta === "object" && !Array.isArray(meta)
? (meta as Record<string, unknown>).mediaImageBlockFactIndexes
: undefined;
if (!Array.isArray(value)) {
return undefined;
}
return value.map((entry) =>
typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0 ? entry : null,
);
}
export function readPersistedPromptMediaFacts(message: AgentMessage): MediaFact[] | undefined {
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
const media =
meta && typeof meta === "object" && !Array.isArray(meta)
? (meta as Record<string, unknown>).media
: undefined;
return Array.isArray(media) ? normalizeMediaFacts(media as MediaFact[]) : undefined;
}
export function readPersistedMediaImageLayout(message: AgentMessage): MediaImageLayout | undefined {
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
if (!meta || typeof meta !== "object" || Array.isArray(meta)) {
return undefined;
}
const layout = (meta as Record<string, unknown>).mediaImageLayout;
if (!layout || typeof layout !== "object" || Array.isArray(layout)) {
return undefined;
}
const record = layout as Record<string, unknown>;
const slots = Array.isArray(record.slots)
? record.slots.flatMap((entry) => {
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
return [];
}
const slot = entry as Record<string, unknown>;
if (slot.kind !== "inline" && slot.kind !== "offloaded") {
return [];
}
const kind: MediaImageLayout["slots"][number]["kind"] = slot.kind;
const factIndex = slot.factIndex;
return [
{
kind,
...(typeof factIndex === "number" && Number.isSafeInteger(factIndex) && factIndex >= 0
? { factIndex }
: {}),
},
];
})
: [];
const suppressedFactIndexes = Array.isArray(record.suppressedFactIndexes)
? record.suppressedFactIndexes.filter(
(entry): entry is number =>
typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0,
)
: [];
return slots.length > 0 || suppressedFactIndexes.length > 0
? { slots, suppressedFactIndexes }
: undefined;
}
@@ -0,0 +1,303 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
import { readRuntimePromptImageFactIndexes } from "../../../media/runtime-prompt-image-provenance.js";
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
import { detectAndLoadPromptImages } from "./images.js";
import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js";
const TINY_PNG_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
describe("plugin harness prompt media", () => {
it("hydrates plugin images and preserves serialized replay order with non-image facts", async () => {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-harness-media-"));
const workspaceDir = path.join(stateDir, "workspace");
const inboundDir = path.join(stateDir, "media", "inbound");
const mediaId = "photo.png";
const imagePath = path.join(inboundDir, mediaId);
await fs.mkdir(workspaceDir, { recursive: true });
await fs.mkdir(inboundDir, { recursive: true });
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
const documentFact = {
path: path.join(workspaceDir, "misleading.png"),
contentType: "application/pdf",
kind: "document" as const,
};
const input = {
runParams: {
agentId: "main",
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
imageOrder: ["offloaded"],
media: [documentFact, { url: `media://inbound/${mediaId}`, contentType: "image/png" }],
sessionId: "session-1",
userTurnTranscriptRecorder: {
message: {
role: "user",
content: "inspect",
MediaPaths: [imagePath, documentFact.path],
MediaTypes: ["image/png", "application/pdf"],
__openclaw: {
mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] },
},
},
},
},
runtime: {
model: { input: ["text", "image"] },
sessionId: "session-1",
workspaceDir,
},
pluginHarnessOwnsTransport: true,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0];
try {
const result = await preparePluginHarnessPromptImages(input);
expect(result.images).toEqual([
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
]);
expect(readRuntimePromptImageFactIndexes(result.images ?? [])).toEqual([0]);
expect(result.imageOrder).toEqual(["inline"]);
expect(result.media?.[0]).toMatchObject({ contentType: "image/png", kind: "image" });
expect(result.media?.[0]).not.toHaveProperty("path");
expect(result.media?.[0]).not.toHaveProperty("url");
expect(result.media?.[1]).toMatchObject(documentFact);
const serialized = JSON.stringify(result);
const restored = JSON.parse(serialized) as typeof result;
const replay = await detectAndLoadPromptImages({
prompt: "",
media: restored.media,
workspaceDir,
model: { input: ["text", "image"] },
existingImages: restored.images,
imageOrder: restored.imageOrder,
});
expect(replay.failedMediaCount).toBe(0);
expect(replay.images).toEqual(result.images);
expect(replay.imageFactIndexes).toEqual([0]);
} finally {
envSnapshot.restore();
await fs.rm(stateDir, { recursive: true, force: true });
}
});
it("surfaces a failed image hydration before plugin dispatch", async () => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-harness-failed-media-"));
try {
await expect(
preparePluginHarnessPromptImages({
runParams: {
agentId: "main",
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
imageOrder: ["offloaded"],
media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }],
sessionId: "session-failed",
},
runtime: {
model: { input: ["text", "image"] },
sessionId: "session-failed",
workspaceDir,
},
pluginHarnessOwnsTransport: true,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
).rejects.toThrow("failed to hydrate 1 structured image attachment");
} finally {
await fs.rm(workspaceDir, { recursive: true, force: true });
}
});
it("surfaces an unsuppressed identity-less inline fact with no image block", async () => {
await expect(
preparePluginHarnessPromptImages({
runParams: {
agentId: "main",
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
imageOrder: ["inline"],
media: [{ kind: "image" }],
sessionId: "session-missing-inline",
},
runtime: {
model: { input: ["text", "image"] },
sessionId: "session-missing-inline",
workspaceDir: "/tmp",
},
pluginHarnessOwnsTransport: true,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
).rejects.toThrow("failed to hydrate 1 structured image attachment");
});
it("surfaces a fact-owned image dropped during host sanitization", async () => {
await expect(
preparePluginHarnessPromptImages({
runParams: {
agentId: "main",
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
images: [{ type: "image", data: "%%%", mimeType: "image/png" }],
imageOrder: ["inline"],
media: [{ kind: "image" }],
sessionId: "session-sanitize-failed",
},
runtime: {
model: { input: ["text", "image"] },
sessionId: "session-sanitize-failed",
workspaceDir: "/tmp",
},
pluginHarnessOwnsTransport: true,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
).rejects.toThrow("failed to hydrate 1 structured image attachment");
});
it("surfaces inline sanitization failure when a preceding plugin image fact is suppressed", async () => {
await expect(
preparePluginHarnessPromptImages({
runParams: {
agentId: "main",
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
images: [{ type: "image", data: "%%%", mimeType: "image/png" }],
imageOrder: ["inline"],
media: [
{
path: "/tmp/described-missing.png",
contentType: "image/png",
hydrationSuppressed: true,
},
{ path: "/tmp/inline.png", contentType: "image/png" },
],
sessionId: "session-suppressed-before-inline",
},
runtime: {
model: { input: ["text", "image"] },
sessionId: "session-suppressed-before-inline",
workspaceDir: "/tmp",
},
pluginHarnessOwnsTransport: true,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
).rejects.toThrow("failed to hydrate 1 structured image attachment");
});
it("retains an intentionally non-hydrating remote-only image as a type-only fact", async () => {
const media = buildInboundMediaNoteProjection({
MediaPaths: [""],
MediaUrls: ["https://example.com/described.png"],
MediaTypes: ["image/png"],
MediaUnderstanding: [
{
kind: "image.description",
attachmentIndex: 0,
text: "already described",
provider: "test",
},
],
}).media;
const result = await preparePluginHarnessPromptImages({
runParams: {
agentId: "main",
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
media,
sessionId: "session-described",
},
runtime: {
model: { input: ["text", "image"] },
sessionId: "session-described",
workspaceDir: "/tmp",
},
pluginHarnessOwnsTransport: true,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
expect(result.images).toEqual([]);
expect(result.media?.[0]).toMatchObject({
contentType: "image/png",
kind: "image",
hydrationSuppressed: true,
});
expect(result.media?.[0]).not.toHaveProperty("path");
expect(result.media?.[0]).not.toHaveProperty("url");
});
it("retains layout-derived suppression after plugin host materialization", async () => {
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
const result = await preparePluginHarnessPromptImages({
runParams: {
agentId: "main",
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
images: [inlineImage],
imageOrder: ["inline"],
media: [
{ path: "/tmp/described.png", contentType: "image/png" },
{ path: "/tmp/inline.png", contentType: "image/png" },
],
sessionId: "session-layout-suppressed",
userTurnTranscriptRecorder: {
message: {
role: "user",
content: "compare",
MediaPaths: ["/tmp/described.png", "/tmp/inline.png"],
MediaTypes: ["image/png", "image/png"],
__openclaw: {
mediaImageLayout: {
slots: [{ kind: "inline", factIndex: 1 }],
suppressedFactIndexes: [0],
},
},
},
},
},
runtime: {
model: { input: ["text", "image"] },
sessionId: "session-layout-suppressed",
workspaceDir: "/tmp",
},
pluginHarnessOwnsTransport: true,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
expect(result.images).toEqual([inlineImage]);
expect(result.imageOrder).toEqual(["inline"]);
expect(result.media?.[0]).toMatchObject({ kind: "image", hydrationSuppressed: true });
expect(result.media?.[1]).toMatchObject({ kind: "image" });
expect(result.media?.[1]).not.toHaveProperty("hydrationSuppressed");
});
it("keeps unsupported native images as aligned type-only facts", async () => {
const media = [
{ path: "/tmp/photo.png", contentType: "image/png" },
{ path: "/tmp/inferred.png", kind: "unknown" as const },
];
const result = await preparePluginHarnessPromptImages({
runParams: {
agentId: "main",
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
media,
sessionId: "session-text-only",
},
runtime: {
model: { input: ["text"] },
sessionId: "session-text-only",
workspaceDir: "/tmp",
},
pluginHarnessOwnsTransport: true,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
expect(result.images).toEqual([]);
expect(result.media?.[0]).toMatchObject({ contentType: "image/png" });
expect(result.media?.[0]).not.toHaveProperty("path");
expect(result.media?.[1]).toMatchObject({ kind: "image" });
expect(result.media?.[1]).not.toHaveProperty("path");
});
it("leaves facts untouched when the native harness owns transport", async () => {
const media = [{ path: "/tmp/photo.png", contentType: "image/png" }];
const result = await preparePluginHarnessPromptImages({
runParams: { media },
runtime: {},
pluginHarnessOwnsTransport: false,
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
expect(result).toEqual({ images: undefined, imageOrder: undefined, media });
});
});
@@ -14,6 +14,7 @@ import {
EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS,
} from "./lane-runtime.js";
import type { RunEmbeddedAgentParams } from "./params.js";
import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js";
import { resolveSkillWorkshopAttemptParams } from "./skill-workshop-attempt-params.js";
import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptTrajectoryRecorder } from "./types.js";
@@ -161,6 +162,11 @@ export async function dispatchEmbeddedRunAttempt(input: {
};
let cancellationRequested = false;
const promptMedia = await preparePluginHarnessPromptImages({
runParams: params,
runtime,
pluginHarnessOwnsTransport: control.pluginHarnessOwnsTransport,
});
const attemptParams: EmbeddedRunAttemptParams = {
operation: "attempt",
sessionId: runtime.sessionId,
@@ -223,8 +229,9 @@ export async function dispatchEmbeddedRunAttempt(input: {
skipPreparedUserTurnMessage: runtime.skipPreparedUserTurnMessage,
currentInboundEventKind: params.currentInboundEventKind,
currentInboundContext: params.currentInboundContext,
images: params.images,
imageOrder: params.imageOrder,
images: promptMedia.images,
imageOrder: promptMedia.imageOrder,
media: promptMedia.media,
clientTools: params.clientTools,
disableTools: params.disableTools,
provider: runtime.provider,
+23 -11
View File
@@ -1,6 +1,8 @@
import { readFileSync } from "node:fs";
import type { ImageContent, TextContent } from "../../llm/types.js";
import { attachRuntimePromptMediaFacts, type MediaFact } from "../../media/media-facts.js";
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
import { readRuntimePromptImageFactIndexes } from "../../media/runtime-prompt-image-provenance.js";
import { attachRuntimeUserTurnTranscriptContext } from "../../sessions/user-turn-transcript-runtime-context.js";
import type {
PersistedUserTurnMessage,
@@ -89,6 +91,21 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
return [{ type: "text", text }, ...(images ?? [])];
}
private createUserMessage(text: string, images?: ImageContent[]): PersistedUserTurnMessage {
const message = {
role: "user",
content: this.createUserContent(text, images),
timestamp: Date.now(),
} satisfies PersistedUserTurnMessage;
const imageFactIndexes = readRuntimePromptImageFactIndexes(images);
return imageFactIndexes
? ({
...message,
__openclaw: { mediaImageBlockFactIndexes: imageFactIndexes },
} as unknown as PersistedUserTurnMessage)
: message;
}
/**
* Send a prompt to the agent.
* - Handles extension commands immediately, even during streaming
@@ -188,11 +205,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
messages = [];
// Add user message
messages.push({
role: "user",
content: this.createUserContent(expandedText, currentImages),
timestamp: Date.now(),
});
messages.push(this.createUserMessage(expandedText, currentImages));
// Inject any pending "nextTurn" messages as context alongside the user message
for (const msg of this.pendingNextTurnMessages) {
@@ -322,6 +335,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
images?: ImageContent[],
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder,
media?: MediaFact[],
imageOrder?: PromptImageOrderEntry[],
): Promise<void> {
// Check for extension commands (cannot be queued)
if (text.startsWith("/")) {
@@ -340,6 +354,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
? { message: preparedMessage, recorder: userTurnTranscriptRecorder }
: undefined,
media,
imageOrder,
);
}
@@ -374,16 +389,13 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
recorder: UserTurnTranscriptRecorder;
},
media?: MediaFact[],
imageOrder?: PromptImageOrderEntry[],
): Promise<void> {
this.steeringMessages.push(text);
this.emitQueueUpdate();
const runtimeMessage = {
role: "user",
content: this.createUserContent(text, images),
timestamp: Date.now(),
} satisfies PersistedUserTurnMessage;
const runtimeMessage = this.createUserMessage(text, images);
const promptMessage = media?.length
? attachRuntimePromptMediaFacts(runtimeMessage, media)
? attachRuntimePromptMediaFacts(runtimeMessage, media, imageOrder)
: runtimeMessage;
this.agent.steer(
transcriptContext
+15 -4
View File
@@ -4,7 +4,9 @@ import { createAssistantMessageEventStream, type AssistantMessage } from "opencl
import { Type } from "typebox";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { getStreamLlmRuntime } from "../../llm/model-runtime-binding.js";
import type { Model, SimpleStreamOptions } from "../../llm/types.js";
import type { ImageContent, Model, SimpleStreamOptions } from "../../llm/types.js";
import { readRuntimePromptImageOrder } from "../../media/media-facts.js";
import { finalizeRuntimePromptImages } from "../../media/runtime-prompt-image-provenance.js";
import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js";
import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js";
@@ -295,8 +297,13 @@ describe("AgentSession queued user turns", () => {
const session = await createSessionFromManager(SessionManager.inMemory());
const steer = vi.spyOn(session.agent, "steer").mockImplementation(() => undefined);
const media = [{ path: "/tmp/a.png", contentType: "image/png" }];
const imageOrder = ["inline"] as const;
const image: ImageContent = { type: "image", data: "aW1hZ2U=", mimeType: "image/png" };
const { images } = finalizeRuntimePromptImages([{ image, factIndex: 0 }]);
await session.steer("[media attached: /tmp/a.png (image/png)]", undefined, undefined, media);
await session.steer("[media attached: /tmp/a.png (image/png)]", images, undefined, media, [
...imageOrder,
]);
const runtimeMessage = steer.mock.calls[0]?.[0];
expect(runtimeMessage).toBeDefined();
@@ -304,12 +311,16 @@ describe("AgentSession queued user turns", () => {
(symbol) => Symbol.keyFor(symbol) === "openclaw.runtimePromptMediaFacts",
);
expect(mediaSymbol).toBeDefined();
if (!mediaSymbol) {
throw new Error("expected runtime prompt media symbol");
if (!runtimeMessage || !mediaSymbol) {
throw new Error("expected runtime prompt media message and symbol");
}
expect((runtimeMessage as unknown as Record<PropertyKey, unknown>)[mediaSymbol]).toEqual([
expect.objectContaining({ path: "/tmp/a.png", contentType: "image/png", kind: "image" }),
]);
expect(readRuntimePromptImageOrder(runtimeMessage)).toEqual(imageOrder);
expect((runtimeMessage as unknown as Record<string, unknown>)["__openclaw"]).toEqual({
mediaImageBlockFactIndexes: [0],
});
expect(JSON.stringify(runtimeMessage)).not.toContain("runtimePromptMediaFacts");
});
});
+13 -2
View File
@@ -111,7 +111,7 @@ describe("buildInboundMediaNote", () => {
});
it("keeps image attachments after image descriptions are added", () => {
const note = buildInboundMediaNote({
const projection = buildInboundMediaNoteProjection({
MediaPaths: ["/tmp/photo.png"],
MediaUrls: ["https://example.com/photo.png"],
MediaTypes: ["image/png"],
@@ -124,9 +124,20 @@ describe("buildInboundMediaNote", () => {
},
],
});
expect(note).toBe(
expect(projection.text).toBe(
"[media attached: /tmp/photo.png (image/png) | https://example.com/photo.png]",
);
expect(projection.media).toEqual([
{
path: "/tmp/photo.png",
url: "https://example.com/photo.png",
contentType: "image/png",
kind: "image",
transcribed: false,
messageId: undefined,
hydrationSuppressed: true,
},
]);
});
it("keeps image attachments when image understanding succeeds via decisions", () => {
+23 -7
View File
@@ -131,12 +131,20 @@ function collectTranscribedAudioAttachmentIndices(
return transcribedAudioIndices;
}
function collectDescribedImageAttachmentIndices(ctx: MsgContext): Set<number> {
return new Set(
ctx.MediaUnderstanding?.flatMap((output) =>
output.kind === "image.description" ? [output.attachmentIndex] : [],
) ?? [],
);
}
type InboundMediaNoteProjection = {
text?: string;
media: MediaFact[];
};
/** Formats prompt-visible attachment text and retains the facts represented by it. */
/** Formats prompt-visible attachment text and retains facts that still need native hydration. */
export function buildInboundMediaNoteProjection(ctx: MsgContext): InboundMediaNoteProjection {
// Attachment indices follow MediaPaths/MediaUrls ordering as supplied by the channel.
const pathsFromArray = Array.isArray(ctx.MediaPaths) ? ctx.MediaPaths : undefined;
@@ -196,14 +204,22 @@ export function buildInboundMediaNoteProjection(ctx: MsgContext): InboundMediaNo
return { media: [] };
}
const facts = resolveMediaFacts(ctx);
const describedImageIndices = collectDescribedImageAttachmentIndices(ctx);
const media = normalizeMediaFacts(
entries.map((entry) => ({
...facts[entry.index],
path: entry.path,
url: entry.url,
contentType: entry.type,
})),
entries.map((entry) =>
Object.assign({}, facts[entry.index], {
path: entry.path,
url: entry.url,
contentType: entry.type,
}),
),
);
for (const [position, entry] of entries.entries()) {
const fact = media[position];
if (fact && describedImageIndices.has(entry.index)) {
fact.hydrationSuppressed = true;
}
}
if (entries.length === 1) {
return {
text: formatMediaAttachedLine({
+1
View File
@@ -1419,6 +1419,7 @@ export async function runReplyAgent(params: {
steeringMode: "all",
isInboundUserMessage: true,
...(followupRun.images?.length ? { images: followupRun.images } : {}),
...(followupRun.imageOrder?.length ? { imageOrder: followupRun.imageOrder } : {}),
...(followupRun.media?.length ? { media: followupRun.media } : {}),
...(turnAdoptionLifecycle ? { waitForTranscriptCommit: true } : {}),
...(resolvedQueue.debounceMs !== undefined ? { debounceMs: resolvedQueue.debounceMs } : {}),
+7 -1
View File
@@ -137,6 +137,7 @@ function appendOrderedImages(params: {
function resolveMergedTurnImages(entries: OrderedTurnImage[]): {
images?: ImageContent[];
imageOrder?: PromptImageOrderEntry[];
imageSourceIndexes?: Array<number | undefined>;
} {
if (entries.length === 0) {
return {};
@@ -151,10 +152,14 @@ function resolveMergedTurnImages(entries: OrderedTurnImage[]): {
return left.sequence - right.sequence;
});
const images = merged.flatMap((entry) => (entry.image ? [entry.image] : []));
return {
const result = {
...(images.length > 0 ? { images } : {}),
imageOrder: merged.map((entry) => entry.imageOrder),
};
Object.defineProperty(result, "imageSourceIndexes", {
value: merged.map((entry) => entry.sourceIndex),
});
return result;
}
/** Resolves current-turn image attachments that were not already described by media understanding. */
@@ -167,6 +172,7 @@ export async function resolveCurrentTurnImages(params: {
}): Promise<{
images?: ImageContent[];
imageOrder?: PromptImageOrderEntry[];
imageSourceIndexes?: Array<number | undefined>;
}> {
const entries: OrderedTurnImage[] = [];
appendOrderedImages({
@@ -1399,6 +1399,13 @@ describe("runPreparedReply media-only handling", () => {
expect(call.followupRun.images).toBeUndefined();
expect(call.followupRun.imageOrder).toBeUndefined();
expect(call.followupRun.prompt).toContain("a tiny dot image");
expect(
(
call.followupRun.userTurnTranscriptRecorder?.message as unknown as Record<string, unknown>
)?.["__openclaw"],
).toMatchObject({
mediaImageLayout: { slots: [], suppressedFactIndexes: [0, 1] },
});
});
it("rehydrates only current MediaPaths missing image understanding", async () => {
@@ -1462,6 +1469,16 @@ describe("runPreparedReply media-only handling", () => {
mimeType: "image/png",
},
]);
expect(
(
call.followupRun.userTurnTranscriptRecorder?.message as unknown as Record<string, unknown>
)?.["__openclaw"],
).toMatchObject({
mediaImageLayout: {
slots: [{ kind: "inline", factIndex: 1 }],
suppressedFactIndexes: [0],
},
});
expect(call.followupRun.imageOrder).toEqual(["inline"]);
expect(call.followupRun.prompt).toContain("a tiny dot image");
});
+79 -2
View File
@@ -43,7 +43,7 @@ import { measureDiagnosticsTimelineSpan } from "../../infra/diagnostics-timeline
import { isFastTestRuntimeEnv } from "../../infra/env.js";
import { resolveHeartbeatRunScope } from "../../infra/heartbeat-run-scope.js";
import type { ExtractedFileImage } from "../../media-understanding/extracted-file-images.js";
import { resolveMediaFacts, type MediaFact } from "../../media/media-facts.js";
import { isImageMediaFact, resolveMediaFacts, type MediaFact } from "../../media/media-facts.js";
import { clearCommandLane, getQueueSize } from "../../process/command-queue.js";
import {
isAcpSessionKey,
@@ -54,6 +54,7 @@ import { MEDIA_ONLY_USER_TEXT } from "../../sessions/user-turn-media.js";
import {
createUserTurnTranscriptRecorder,
resolvePersistedUserTurnText,
type UserTurnInput,
} from "../../sessions/user-turn-transcript.js";
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
import type { SilentReplyConversationType } from "../../shared/silent-reply-policy.js";
@@ -162,6 +163,73 @@ type InternalGetReplyOptions = BaseInternalGetReplyOptions & {
type AgentDefaults = NonNullable<OpenClawConfig["agents"]>["defaults"];
type ExecOverrides = Pick<ExecToolDefaults, "host" | "security" | "ask" | "node" | "nodeCwd">;
const EPOCH_MILLISECONDS_THRESHOLD = 1_000_000_000_000;
function buildPersistedMediaImageLayout(params: {
ctx: MsgContext;
media: readonly MediaFact[];
ctxMediaCount: number;
imageOrder?: readonly ("inline" | "offloaded")[];
imageSourceIndexes?: readonly (number | undefined)[];
}): NonNullable<UserTurnInput["mediaImageLayout"]> | undefined {
const describedAttachmentIndexes = new Set(
params.ctx.MediaUnderstanding?.flatMap((output) =>
output.kind === "image.description" ? [output.attachmentIndex] : [],
) ?? [],
);
const suppressedFactIndexes: number[] = [];
const imageFactIndexes: number[] = [];
for (const [factIndex, fact] of params.media.entries()) {
if (!isImageMediaFact(fact)) {
continue;
}
imageFactIndexes.push(factIndex);
if (
(factIndex < params.ctxMediaCount && describedAttachmentIndexes.has(factIndex)) ||
fact.hydrationSuppressed === true
) {
suppressedFactIndexes.push(factIndex);
}
}
if (imageFactIndexes.length === 0) {
return undefined;
}
const suppressed = new Set(suppressedFactIndexes);
const used = new Set<number>();
const unsuppressedFactCount = imageFactIndexes.filter((index) => !suppressed.has(index)).length;
const canInferByPosition = unsuppressedFactCount === (params.imageOrder?.length ?? 0);
const takeNextFactIndex = (): number | undefined =>
imageFactIndexes.find((index) => !suppressed.has(index) && !used.has(index));
const slots = (params.imageOrder ?? []).map((kind, index) => {
const sourceIndex = params.imageSourceIndexes?.[index];
const sourceFact = sourceIndex === undefined ? undefined : params.media[sourceIndex];
const factIndex =
sourceIndex !== undefined
? sourceFact &&
isImageMediaFact(sourceFact) &&
!suppressed.has(sourceIndex) &&
!used.has(sourceIndex)
? sourceIndex
: undefined
: canInferByPosition
? takeNextFactIndex()
: undefined;
if (factIndex !== undefined) {
used.add(factIndex);
}
return factIndex === undefined ? { kind } : { kind, factIndex };
});
for (const factIndex of imageFactIndexes) {
if (!suppressed.has(factIndex) && !used.has(factIndex)) {
slots.push({ kind: "offloaded", factIndex });
}
}
if (slots.length === 0 && suppressedFactIndexes.length === 0) {
return undefined;
}
return {
slots,
...(suppressedFactIndexes.length > 0 ? { suppressedFactIndexes } : {}),
};
}
function hasResolvedThinkingCatalogEntry(params: {
catalog?: readonly ThinkingCatalogEntry[];
@@ -1466,7 +1534,15 @@ export async function runPreparedReply(
: undefined);
setChannelSourceTurnId(sessionCtx, sourceTurnId);
const persistGroupSender = replyRoute.chatType === "group" || replyRoute.chatType === "channel";
const userTurnMediaForPersistence = [...resolveMediaFacts(ctx), ...(opts?.media ?? [])];
const ctxMediaForPersistence = resolveMediaFacts(ctx);
const userTurnMediaForPersistence = [...ctxMediaForPersistence, ...(opts?.media ?? [])];
const mediaImageLayout = buildPersistedMediaImageLayout({
ctx,
media: userTurnMediaForPersistence,
ctxMediaCount: ctxMediaForPersistence.length,
imageOrder: currentTurnImages.imageOrder,
imageSourceIndexes: currentTurnImages.imageSourceIndexes,
});
const inputProvenance = ctx.InputProvenance ?? sessionCtx.InputProvenance;
const userTurnTimestamp = normalizeMessageTimestampMs(ctx.Timestamp);
// prompt-prelude substitutes MEDIA_ONLY_USER_TEXT as transcriptBody for
@@ -1518,6 +1594,7 @@ export async function runPreparedReply(
: {}),
...(transport ? { transport } : {}),
...(userTurnMediaForPersistence.length > 0 ? { media: userTurnMediaForPersistence } : {}),
...(mediaImageLayout ? { mediaImageLayout } : {}),
// Persist the message's own arrival timestamp so the single
// LLM-boundary stamping site (normalizeMessagesForLlmBoundary) can
// derive a stable per-message `[DOW YYYY-MM-DD HH:MM TZ]` prefix that
@@ -18,6 +18,7 @@ import {
} from "../../logging/diagnostic-run-activity.js";
import { diagnosticLogger as diag } from "../../logging/diagnostic-runtime.js";
import type { MediaFact } from "../../media/media-facts.js";
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.types.js";
import { resolveGlobalSingleton } from "../../shared/global-singleton.js";
import { resolveTimerTimeoutMs } from "../../shared/number-coercion.js";
@@ -41,6 +42,7 @@ export type ReplyBackendQueueMessageOptions = {
debounceMs?: number;
/** Ordered current-turn images to inject with the steering text. */
images?: ImageContent[];
imageOrder?: PromptImageOrderEntry[];
/** Ordered facts represented by attachment text in this steering prompt. */
media?: MediaFact[];
deliveryTimeoutMs?: number;
+25 -15
View File
@@ -232,10 +232,8 @@ describe("parseMessageWithAttachments", () => {
expect(ref.mimeType).toBe("application/pdf");
expect(ref.label).toBe("report.pdf");
expect(ref.mediaRef).toMatch(/^media:\/\/inbound\//);
// Non-image offloads MUST NOT inject a media://URI into the message —
// the caller is responsible for routing offloadedRefs[].path into
// ctx.MediaPaths so the workspace stage surfaces a real path.
expect(parsed.message).toBe("read this");
expect(parsed.message).toBe(`read this\n[media attached: ${ref.mediaRef}]`);
expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message);
expect(saveMediaBufferMock).toHaveBeenCalledOnce();
expect(savedMime()).toBe("application/pdf");
expect(logs).toHaveLength(0);
@@ -251,7 +249,10 @@ describe("parseMessageWithAttachments", () => {
expect(parsed.offloadedRefs).toHaveLength(1);
expect(parsed.offloadedRefs[0]?.mimeType).toBe("application/octet-stream");
expect(savedMime()).toBe("application/octet-stream");
expect(parsed.message).toBe("take a look");
expect(parsed.message).toBe(
`take a look\n[media attached: ${parsed.offloadedRefs[0]?.mediaRef}]`,
);
expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message);
expect(logs).toHaveLength(0);
});
@@ -272,15 +273,12 @@ describe("parseMessageWithAttachments", () => {
expect(parsed.imageOrder).toEqual(["inline"]);
});
it("excludes non-image offloads from imageOrder in mixed batches", async () => {
it("keeps mixed image/PDF markers in normal and image-stripped routing order", async () => {
// Regression: a prior revision pushed "offloaded" for every offload,
// including non-image files. In a [non-image, inline, offloaded-image]
// batch that produced imageOrder=["offloaded","inline","offloaded"] even
// though only one `[media attached: media://...]` line is ever appended
// to the prompt (for the image offload). extractTrailingAttachmentMediaUris
// then read count=2 against one trailing URI, and
// mergePromptAttachmentImages placed the single offloaded image into the
// first "offloaded" slot — swapping it ahead of the inline image.
// though only one image offload existed. Structural facts and imageOrder
// must agree so hydration cannot swap it ahead of the inline image.
const pdf = Buffer.from("%PDF-1.4\n").toString("base64");
const bigPng = Buffer.alloc(2_100_000);
bigPng.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0);
@@ -295,12 +293,16 @@ describe("parseMessageWithAttachments", () => {
"image/png",
]);
expect(parsed.imageOrder).toEqual(["inline", "offloaded"]);
// The offloaded-image URI is the sole trailing media:// line, matching
// imageOrder's single "offloaded" slot.
const pdfRef = expectDefined(parsed.offloadedRefs[0], "offloaded PDF ref");
const imageRef = expectDefined(parsed.offloadedRefs[1], "offloaded image ref");
expect(parsed.message).toBe(
`x\n[media attached: ${pdfRef.mediaRef}]\n[media attached: ${imageRef.mediaRef}]`,
);
expect(parsed.messageWithoutOffloadedImageRefs).toBe(`x\n[media attached: ${pdfRef.mediaRef}]`);
const trailingMediaLines = parsed.message
.split("\n")
.filter((line) => line.trim().startsWith("[media attached: media://inbound/"));
expect(trailingMediaLines).toHaveLength(1);
expect(trailingMediaLines).toHaveLength(2);
});
it("rejects oversized images before offload", async () => {
@@ -443,6 +445,7 @@ describe("parseMessageWithAttachments validation errors", () => {
it("passes through unchanged on text-only session with no attachments", async () => {
const { parsed } = await parseWithWarnings("hello", [], { supportsInlineImages: false });
expect(parsed.message).toBe("hello");
expect(parsed.messageWithoutOffloadedImageRefs).toBe("hello");
expect(parsed.images).toHaveLength(0);
expect(parsed.offloadedRefs).toHaveLength(0);
expect(saveMediaBufferMock).not.toHaveBeenCalled();
@@ -468,7 +471,10 @@ describe("parseMessageWithAttachments validation errors", () => {
expect(parsed.offloadedRefs).toHaveLength(1);
expect(parsed.offloadedRefs[0]?.mimeType).toBe("application/pdf");
expect(parsed.offloadedRefs[0]?.label).toBe("brief.pdf");
expect(parsed.message).toBe("read this");
expect(parsed.message).toBe(
`read this\n[media attached: ${parsed.offloadedRefs[0]?.mediaRef}]`,
);
expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message);
} finally {
await cleanupOffloadedRefs(parsed.offloadedRefs);
}
@@ -495,6 +501,7 @@ describe("parseMessageWithAttachments validation errors", () => {
const offloaded = expectDefined(parsed.offloadedRefs[0], "offloaded image ref");
expect(offloaded.mimeType).toBe("image/png");
expect(parsed.message).toBe(`see this\n[media attached: ${offloaded.mediaRef}]`);
expect(parsed.messageWithoutOffloadedImageRefs).toBe("see this");
expect(parsed.media).toEqual([
{
path: offloaded.path,
@@ -530,6 +537,9 @@ describe("parseMessageWithAttachments validation errors", () => {
expect(parsed.message).toContain(
"[image attachment omitted: text-only attachment limit reached]",
);
expect(parsed.messageWithoutOffloadedImageRefs).toBe(
"see these\n[image attachment omitted: text-only attachment limit reached]",
);
expect(logs).toEqual([
"attachment dot-10.png: dropping image because text-only offload limit 10 was reached",
]);
+20 -3
View File
@@ -36,6 +36,7 @@ export type OffloadedRef = {
type ParsedMessageWithImages = {
message: string;
messageWithoutOffloadedImageRefs: string;
images: ChatImageContent[];
imageOrder: PromptImageOrderEntry[];
media: MediaFact[];
@@ -322,13 +323,21 @@ export async function parseMessageWithAttachments(
const acceptNonImage = opts?.acceptNonImage !== false;
if (!attachments || attachments.length === 0) {
return { message, images: [], imageOrder: [], media: [], offloadedRefs: [] };
return {
message,
messageWithoutOffloadedImageRefs: message,
images: [],
imageOrder: [],
media: [],
offloadedRefs: [],
};
}
const images: ChatImageContent[] = [];
const imageOrder: PromptImageOrderEntry[] = [];
const offloadedRefs: OffloadedRef[] = [];
let updatedMessage = message;
let messageWithoutOffloadedImageRefs = message;
let textOnlyImageOffloadCount = 0;
const savedMediaIds: string[] = [];
@@ -422,6 +431,8 @@ export async function parseMessageWithAttachments(
`${TEXT_ONLY_OFFLOAD_LIMIT} was reached`,
);
updatedMessage += "\n[image attachment omitted: text-only attachment limit reached]";
messageWithoutOffloadedImageRefs +=
"\n[image attachment omitted: text-only attachment limit reached]";
continue;
}
@@ -458,8 +469,10 @@ export async function parseMessageWithAttachments(
savedMediaIds.push(savedMedia.id);
const mediaRef = `media://inbound/${savedMedia.id}`;
if (isImage) {
updatedMessage += `\n[media attached: ${mediaRef}]`;
const mediaLine = `\n[media attached: ${mediaRef}]`;
updatedMessage += mediaLine;
if (!isImage) {
messageWithoutOffloadedImageRefs += mediaLine;
}
log?.info?.(
shouldForceImageOffload && isImage
@@ -491,6 +504,10 @@ export async function parseMessageWithAttachments(
return {
message: updatedMessage !== message ? updatedMessage.trimEnd() : message,
messageWithoutOffloadedImageRefs:
messageWithoutOffloadedImageRefs !== message
? messageWithoutOffloadedImageRefs.trimEnd()
: message,
images,
imageOrder,
media: offloadedRefs.map((ref) => ({
@@ -57,23 +57,6 @@ function logAttachmentFailure(
});
}
function stripTrailingOffloadedMediaMarkers(message: string, refs: OffloadedRef[]): string {
if (refs.length === 0) {
return message;
}
const removableRefs = new Set(refs.map((ref) => ref.mediaRef));
const lines = message.split(/\r?\n/);
while (lines.length > 0) {
const last = lines[lines.length - 1]?.trim() ?? "";
const match = /^\[media attached:\s*(media:\/\/inbound\/[^\]\s]+)\]$/.exec(last);
if (!match?.[1] || !removableRefs.delete(match[1])) {
break;
}
lines.pop();
}
return lines.join("\n").trimEnd();
}
function isPdfOffloadedRef(ref: OffloadedRef): boolean {
const mime = ref.mimeType.trim().toLowerCase();
if (mime === "application/pdf" || mime.endsWith("+pdf")) {
@@ -263,14 +246,11 @@ export async function prepareChatSendAttachments(params: {
supportsImages,
acceptNonImage: true,
});
parsedMessage = stripTrailingOffloadedMediaMarkers(
parsed.message,
routeImageOffloadsAsMediaPaths
? parsed.offloadedRefs.filter((ref) => ref.mimeType.startsWith("image/"))
: [],
);
parsedMessage = routeImageOffloadsAsMediaPaths
? parsed.messageWithoutOffloadedImageRefs
: parsed.message;
parsedImages = parsed.images;
imageOrder = routeImageOffloadsAsMediaPaths ? [] : parsed.imageOrder;
imageOrder = parsed.imageOrder;
offloadedRefs = parsed.offloadedRefs;
({
paths: mediaPathOffloadPaths,
@@ -1,11 +1,21 @@
import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import {
GATEWAY_CLIENT_IDS,
GATEWAY_CLIENT_MODES,
type GatewayClientInfo,
} from "../../../packages/gateway-protocol/src/client-info.js";
import { createSolidPngBuffer } from "../../../test/helpers/image-fixtures.js";
import { pruneProcessedHistoryImages } from "../../agents/embedded-agent-runner/run/history-image-prune.js";
import { hydratePromptMediaMessages } from "../../agents/embedded-agent-runner/run/images.js";
import type { AgentMessage } from "../../agents/runtime/index.js";
import type { MsgContext } from "../../auto-reply/templating.js";
import type { UserTurnInput } from "../../sessions/user-turn-transcript.js";
import { resolveStateDir } from "../../config/paths.js";
import {
buildPersistedUserTurnMessage,
type UserTurnInput,
} from "../../sessions/user-turn-transcript.js";
import { applyChatSendManagedMediaFields, prepareChatSendUserTurn } from "./chat-send-user-turn.js";
function createUserTurnInputController() {
@@ -206,8 +216,8 @@ describe("prepareChatSendUserTurn", () => {
await expect(readInput()).resolves.toEqual(controller.baseInput);
});
it("carries retained image claim-check facts without changing the trailing prompt line", () => {
const { controller } = createUserTurnInputController();
it("carries retained image claim-check facts without changing the trailing prompt line", async () => {
const { controller, readInput } = createUserTurnInputController();
const mediaRef = "media://inbound/image-1.png";
const prepared = prepareChatSendUserTurn({
request: {
@@ -255,6 +265,166 @@ describe("prepareChatSendUserTurn", () => {
contentType: "image/png",
},
]);
await expect(readInput()).resolves.toMatchObject({
mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] },
});
});
it("persists and prunes the staged PDF claim-check alias as structured ownership", async () => {
const { controller, readInput } = createUserTurnInputController();
const mediaRef = "media://inbound/report.pdf";
prepareChatSendUserTurn({
request: {
clientInfo: createClientInfo(),
normalizedAttachments: [{}],
suppressCommandInterpretation: false,
systemInputProvenance: undefined,
systemProvenanceReceipt: undefined,
},
session: {
agentId: "main",
clientRunId: "run-1",
sessionKey: "agent:main:main",
},
admission: {
originatingRoute: { originatingChannel: "webchat", explicitDeliverRoute: false },
},
attachments: createAttachments({
offloadedRefs: [
{
mediaRef,
id: "report.pdf",
path: "/media/inbound/report.pdf",
mimeType: "application/pdf",
label: "report.pdf",
sizeBytes: 10,
},
],
parsedMessage: `read this\n[media attached: ${mediaRef}]`,
}),
client: null,
logGateway: { warn: vi.fn() } as never,
userTurn: controller,
});
const input = await readInput();
expect(input.media).toEqual([
{
path: "/media/inbound/report.pdf",
url: mediaRef,
contentType: "application/pdf",
hydrationSuppressed: true,
},
]);
const persisted = buildPersistedUserTurnMessage({
...input,
text: `read this\n[media attached: ${mediaRef}]`,
});
const history = [
persisted,
{ role: "assistant", content: "ack" },
{ role: "user", content: "more" },
{ role: "assistant", content: "ack" },
{ role: "user", content: "more" },
{ role: "assistant", content: "ack" },
{ role: "user", content: "more" },
{ role: "assistant", content: "ack" },
] as unknown as Parameters<typeof pruneProcessedHistoryImages>[0];
const pruned = pruneProcessedHistoryImages(history);
const first = pruned?.[0] as unknown as Record<string, unknown> | undefined;
expect(first?.content).toBe(
"read this\n[media reference removed - already processed by model]",
);
expect((first?.["__openclaw"] as Record<string, unknown> | undefined)?.media).toBeUndefined();
});
it("hydrates and prunes a staged image claim-check alias as structured ownership", async () => {
const id = `gateway-image-${Date.now()}-${Math.random().toString(36).slice(2)}.png`;
const imagePath = path.join(resolveStateDir(), "media", "inbound", id);
const mediaRef = `media://inbound/${id}`;
const unownedRef = "media://inbound/unowned.png";
const text = `inspect\n[media attached: ${mediaRef}]\n[media attached: ${unownedRef}]`;
await fs.mkdir(path.dirname(imagePath), { recursive: true });
await fs.writeFile(imagePath, createSolidPngBuffer(2, 2, { r: 10, g: 20, b: 30 }));
try {
const { controller, readInput } = createUserTurnInputController();
prepareChatSendUserTurn({
request: {
clientInfo: createClientInfo(),
normalizedAttachments: [{}],
suppressCommandInterpretation: false,
systemInputProvenance: undefined,
systemProvenanceReceipt: undefined,
},
session: {
agentId: "main",
clientRunId: "run-1",
sessionKey: "agent:main:main",
},
admission: {
originatingRoute: { originatingChannel: "webchat", explicitDeliverRoute: false },
},
attachments: createAttachments({
imageOrder: ["offloaded"],
offloadedRefs: [
{
mediaRef,
id,
path: imagePath,
mimeType: "image/png",
label: "image.png",
sizeBytes: 10,
},
],
parsedMessage: text,
}),
client: null,
logGateway: { warn: vi.fn() } as never,
userTurn: controller,
});
const input = await readInput();
expect(input.media).toEqual([{ path: imagePath, url: mediaRef, contentType: "image/png" }]);
expect(input.media?.[0]).not.toHaveProperty("hydrationSuppressed");
const persisted = buildPersistedUserTurnMessage({ ...input, text });
expect(
(
(persisted as unknown as Record<string, unknown>)["__openclaw"] as {
media?: unknown;
}
).media,
).toEqual([{ path: imagePath, url: mediaRef, contentType: "image/png" }]);
const hydrated = await hydratePromptMediaMessages([persisted as AgentMessage], {
workspaceDir: path.dirname(imagePath),
model: { input: ["text", "image"] },
workspaceOnly: true,
});
expect((hydrated[0] as unknown as { content?: unknown[] }).content).toEqual([
{ type: "text", text },
expect.objectContaining({ type: "image", mimeType: "image/png" }),
]);
const history = [
persisted,
{ role: "assistant", content: "ack" },
{ role: "user", content: "more" },
{ role: "assistant", content: "ack" },
{ role: "user", content: "more" },
{ role: "assistant", content: "ack" },
{ role: "user", content: "more" },
{ role: "assistant", content: "ack" },
] as unknown as Parameters<typeof pruneProcessedHistoryImages>[0];
const pruned = pruneProcessedHistoryImages(history);
const first = pruned?.[0] as unknown as Record<string, unknown> | undefined;
expect(first?.content).toBe(
`inspect\n[media reference removed - already processed by model]\n[media attached: ${unownedRef}]`,
);
expect((first?.["__openclaw"] as Record<string, unknown> | undefined)?.media).toBeUndefined();
} finally {
await fs.rm(imagePath, { force: true });
}
});
});
@@ -93,11 +93,26 @@ export function applyChatSendManagedMediaFields(
}
}
function buildChatSendUserTurnMedia(savedMedia: SavedMedia[]): NonNullable<UserTurnInput["media"]> {
return savedMedia.map((entry) => ({
path: entry.path,
contentType: entry.contentType,
}));
function buildChatSendUserTurnMedia(
savedMedia: SavedMedia[],
offloadedRefs: OffloadedRef[],
): NonNullable<UserTurnInput["media"]> {
const offloadedRefsById = new Map(offloadedRefs.map((ref) => [ref.id, ref] as const));
return savedMedia.map((entry) => {
const offloadedRef = offloadedRefsById.get(entry.id);
return {
path: entry.path,
...(offloadedRef
? {
// Every offload keeps its claim-check alias so persisted marker
// ownership survives; only non-images skip native image hydration.
url: offloadedRef.mediaRef,
...(offloadedRef.mimeType.startsWith("image/") ? {} : { hydrationSuppressed: true }),
}
: {}),
contentType: entry.contentType,
};
});
}
function buildChatSendPromptMedia(
@@ -251,10 +266,21 @@ export function prepareChatSendUserTurn(params: {
? getPersistedMediaForTranscript()
: Promise.resolve([]);
userTurn.setInputPromise(
preparedUserTurnMediaPromise.then(buildChatSendUserTurnMedia).then((media) => ({
...userTurn.baseInput,
...(media.length > 0 ? { media } : {}),
})),
preparedUserTurnMediaPromise
.then((media) => buildChatSendUserTurnMedia(media, attachments.offloadedRefs))
.then((media) => ({
...userTurn.baseInput,
...(media.length > 0 ? { media } : {}),
...(media.length > 0 && attachments.imageOrder.length > 0
? {
mediaImageLayout: {
// persistInboundImagesForTranscript emits image facts in this exact order,
// then appends non-images, so image slot ordinals are fact ordinals.
slots: attachments.imageOrder.map((kind, factIndex) => ({ kind, factIndex })),
},
}
: {}),
})),
);
const pluginBoundMediaFieldsPromise =
attachments.explicitOriginTargetsPlugin && attachments.parsedImages.length > 0
@@ -5256,7 +5256,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
| undefined;
expect(mockState.lastDispatchImages).toBeUndefined();
expect(mockState.lastDispatchImageOrder).toBeUndefined();
expect(mockState.lastDispatchCtx?.Body).toBe("summarize this");
expect(mockState.lastDispatchCtx?.Body).toBe(
"summarize this\n[media attached: media://inbound/saved-media]",
);
expect(mockState.savedMediaCalls[0]?.contentType).toBe("application/pdf");
expect(mockState.savedMediaCalls[0]?.subdir).toBe("inbound");
expect(typeof mockState.savedMediaCalls[0]?.size).toBe("number");
@@ -5635,7 +5637,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
});
expect(mockState.lastDispatchImages).toBeUndefined();
expect(mockState.lastDispatchImageOrder).toBeUndefined();
expect(mockState.lastDispatchImageOrder).toEqual(["offloaded"]);
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
expect(mockState.lastDispatchCtx?.Body).not.toContain("media://");
expect(mockState.lastDispatchCtx?.MediaPath).toBe("/tmp/1.png");
@@ -5811,7 +5813,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
});
expect(mockState.lastDispatchImages).toBeUndefined();
expect(mockState.lastDispatchImageOrder).toBeUndefined();
expect(mockState.lastDispatchImageOrder).toEqual(["offloaded"]);
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
expect(mockState.lastDispatchCtx?.Body).not.toContain("media://");
expect(mockState.lastDispatchCtx?.MediaPath).toBe("/tmp/1.png");
@@ -5876,11 +5878,14 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
);
expect(mockState.lastDispatchCtx?.MediaTypes).toEqual(["application/pdf"]);
expect(mockState.lastDispatchCtx?.MediaType).toBe("application/pdf");
// Non-image offloads MUST NOT inject a media://URI into the prompt body —
// they ride through ctx.MediaPaths so buildInboundMediaNote prepends the
// real path, avoiding duplicate media markers.
expect(mockState.lastDispatchCtx?.Body).not.toContain("media://");
expect(mockState.lastDispatchCtx?.BodyForAgent).not.toContain("media://");
// Non-image offloads retain their claim-check line while the staged path
// also travels structurally for media tools and transcript persistence.
expect(mockState.lastDispatchCtx?.Body).toContain(
"[media attached: media://inbound/saved-media]",
);
expect(mockState.lastDispatchCtx?.BodyForAgent).toContain(
"[media attached: media://inbound/saved-media]",
);
expect(mockState.lastDispatchImages).toBeUndefined();
// Marker replaces the implicit "relative-path no-op" coupling in
// get-reply.ts with an explicit skip contract.
@@ -5939,7 +5944,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
]);
expect(mockState.lastDispatchCtx?.MediaTypes).toEqual(["application/zip"]);
expect(mockState.lastDispatchImages).toBeUndefined();
expect(mockState.lastDispatchCtx?.Body).not.toContain("media://");
expect(mockState.lastDispatchCtx?.Body).toContain(
"[media attached: media://inbound/saved-media]",
);
expect(mockState.lastDispatchCtx?.MediaStaged).toBe(true);
});
+63 -3
View File
@@ -1,6 +1,7 @@
import type { MediaKind } from "@openclaw/media-core/constants";
import { kindFromMime } from "@openclaw/media-core/mime";
import { kindFromMime, mimeTypeFromFilePath } from "@openclaw/media-core/mime";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import type { PromptImageOrderEntry } from "./prompt-image-order.js";
/** One ordered runtime attachment; array position is its alignment identity. */
export type MediaFact = {
@@ -11,6 +12,10 @@ export type MediaFact = {
transcribed?: boolean;
messageId?: string;
workspaceDir?: string;
// Declared field, not a symbol: suppression must survive every fact copy or
// reprojection boundary; described images otherwise rehydrate or count failed.
// Structured persistence may retain it; legacy Media* projections never emit it.
hydrationSuppressed?: boolean;
};
export type MediaFactInput = {
@@ -23,14 +28,66 @@ const RUNTIME_PROMPT_MEDIA_FACTS = Symbol.for("openclaw.runtimePromptMediaFacts"
export function attachRuntimePromptMediaFacts<T extends object>(
message: T,
media: readonly MediaFact[],
imageOrder?: readonly PromptImageOrderEntry[],
): T {
const normalized = normalizeMediaFacts(media);
if (imageOrder?.length) {
Object.defineProperty(normalized, "imageOrder", { value: [...imageOrder] });
}
Object.defineProperty(message, RUNTIME_PROMPT_MEDIA_FACTS, {
configurable: true,
value: normalizeMediaFacts(media),
value: normalized,
});
return message;
}
export function readRuntimePromptMediaFacts(message: object): MediaFact[] | undefined {
const media = (message as Record<PropertyKey, unknown>)[RUNTIME_PROMPT_MEDIA_FACTS];
return Array.isArray(media) ? (media as MediaFact[]) : undefined;
}
export function readRuntimePromptImageOrder(message: object): PromptImageOrderEntry[] | undefined {
const imageOrder = (
readRuntimePromptMediaFacts(message) as
| (MediaFact[] & { imageOrder?: PromptImageOrderEntry[] })
| undefined
)?.imageOrder;
return Array.isArray(imageOrder) ? (imageOrder as PromptImageOrderEntry[]) : undefined;
}
/** Returns whether a fact can produce native image input. */
export function isImageMediaFact(fact: MediaFactInput): boolean {
if (fact.kind && fact.kind !== "unknown") {
return fact.kind === "image" || fact.kind === "sticker";
}
const contentType = normalizeOptionalString(fact.contentType);
const normalizedContentType = contentType?.split(";")[0]?.trim().toLowerCase();
if (
normalizedContentType &&
normalizedContentType !== "application/octet-stream" &&
normalizedContentType !== "binary/octet-stream"
) {
const mimeKind = kindFromMime(normalizedContentType);
if (mimeKind) {
return mimeKind === "image";
}
// Legacy channel-mode projections persist bare kinds as MediaType; honor
// them, and fall through to filename inference for other unknown strings.
if (normalizedContentType === "image" || normalizedContentType === "sticker") {
return true;
}
if (
normalizedContentType === "audio" ||
normalizedContentType === "video" ||
normalizedContentType === "document"
) {
return false;
}
}
const pathValue = normalizeOptionalString(fact.path) ?? normalizeOptionalString(fact.url);
return kindFromMime(mimeTypeFromFilePath(pathValue)) === "image";
}
type MediaFactDefaults<TInput extends MediaFactInput = MediaFactInput> = {
kind?: MediaKind;
messageId?: string;
@@ -61,7 +118,7 @@ function normalizeMediaFact<TInput extends MediaFactInput>(
): MediaFact {
const workspaceDir = normalizeOptionalString(media.workspaceDir) ?? defaults.workspaceDir;
const contentType = normalizeOptionalString(media.contentType);
return {
const normalized: MediaFact = {
path: normalizeOptionalString(media.path),
url: normalizeOptionalString(media.url),
contentType,
@@ -69,7 +126,9 @@ function normalizeMediaFact<TInput extends MediaFactInput>(
transcribed: media.transcribed === true || defaults.transcribed?.(media, index) === true,
messageId: normalizeOptionalString(media.messageId) ?? defaults.messageId,
...(workspaceDir ? { workspaceDir } : {}),
...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}),
};
return normalized;
}
/** True when a consumer must use the already-staged legacy path projection. */
@@ -136,6 +195,7 @@ export function resolveMediaFacts(source: MediaFactSource): MediaFact[] {
workspaceDir:
normalizeOptionalString(fact?.workspaceDir) ??
normalizeOptionalString(source.MediaWorkspaceDir),
hydrationSuppressed: fact?.hydrationSuppressed,
},
index,
);
@@ -0,0 +1,45 @@
const RUNTIME_PROMPT_IMAGE_FACT_INDEXES = Symbol.for("openclaw.runtimePromptImageFactIndexes");
type RuntimePromptImageFactIndex = number | null;
export function finalizeRuntimePromptImages<TImage extends object>(
entries: readonly { image: TImage; factIndex: RuntimePromptImageFactIndex }[],
): { images: TImage[]; imageFactIndexes: RuntimePromptImageFactIndex[] } {
const images = entries.map((entry) => entry.image);
const imageFactIndexes = entries.map((entry) => entry.factIndex);
attachRuntimePromptImageFactIndexes(images, imageFactIndexes);
return { images, imageFactIndexes };
}
/** Carries fact ownership on image blocks without changing provider-visible bytes. */
function attachRuntimePromptImageFactIndexes(
images: readonly object[],
factIndexes: readonly RuntimePromptImageFactIndex[],
): void {
if (images.length !== factIndexes.length) {
return;
}
Object.defineProperty(images, RUNTIME_PROMPT_IMAGE_FACT_INDEXES, {
configurable: true,
value: [...factIndexes],
});
}
export function readRuntimePromptImageFactIndexes(
images: readonly object[] | null | undefined,
): RuntimePromptImageFactIndex[] | undefined {
if (!images?.length) {
return undefined;
}
const factIndexes = (images as unknown as Record<PropertyKey, unknown>)[
RUNTIME_PROMPT_IMAGE_FACT_INDEXES
];
return Array.isArray(factIndexes) &&
factIndexes.length === images.length &&
factIndexes.every(
(entry) =>
entry === null || (typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0),
)
? (factIndexes as RuntimePromptImageFactIndex[])
: undefined;
}
@@ -0,0 +1,97 @@
import path from "node:path";
import { mimeTypeFromFilePath } from "@openclaw/media-core/mime";
import type { MediaFactInput } from "../media/media-facts.js";
import type { PersistedUserTurnMediaInput } from "./user-turn-transcript.types.js";
const URL_LIKE_MEDIA_PATH_PATTERN = /^[a-z][a-z0-9+.-]*:/i;
const STRUCTURED_MEDIA_KINDS = new Set<NonNullable<MediaFactInput["kind"]>>([
"image",
"audio",
"video",
"document",
"sticker",
"unknown",
]);
function normalizeOptionalText(value: string | null | undefined): string | undefined {
const normalized = value?.trim();
return normalized ? normalized : undefined;
}
function mediaTypeForTranscript(media: PersistedUserTurnMediaInput, mediaPath?: string): string {
return (
normalizeOptionalText(media.contentType) ??
normalizeOptionalText(media.kind) ??
mimeTypeFromFilePath(mediaPath) ??
"application/octet-stream"
);
}
function normalizeStructuredMediaKind(value: string | null | undefined): MediaFactInput["kind"] {
const kind = normalizeOptionalText(value);
return kind && STRUCTURED_MEDIA_KINDS.has(kind as NonNullable<MediaFactInput["kind"]>)
? (kind as NonNullable<MediaFactInput["kind"]>)
: undefined;
}
export function resolveTranscriptMediaPath(
pathValue: string,
workspaceDir: string | undefined,
): string {
// Relative staged media paths are anchored to the media workspace; absolute
// paths and URL-like refs are already stable transcript references.
if (!workspaceDir || path.isAbsolute(pathValue) || URL_LIKE_MEDIA_PATH_PATTERN.test(pathValue)) {
return pathValue;
}
return path.join(workspaceDir, pathValue);
}
export function normalizeMediaEntryForTranscript(
media: PersistedUserTurnMediaInput,
): MediaFactInput {
const rawPath = normalizeOptionalText(media.path) ?? normalizeOptionalText(media.url);
if (!rawPath) {
return media.hydrationSuppressed === true
? {
contentType: normalizeOptionalText(media.contentType),
hydrationSuppressed: true,
}
: {};
}
return {
path: resolveTranscriptMediaPath(rawPath, normalizeOptionalText(media.workspaceDir)),
contentType: mediaTypeForTranscript(media, rawPath),
...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}),
};
}
export function normalizeStructuredMediaEntryForTranscript(
media: PersistedUserTurnMediaInput,
): MediaFactInput {
const mediaPath = normalizeOptionalText(media.path);
const mediaUrl = normalizeOptionalText(media.url);
return {
path: mediaPath,
url: mediaUrl,
contentType:
normalizeOptionalText(media.contentType) ?? mimeTypeFromFilePath(mediaPath ?? mediaUrl),
kind: normalizeStructuredMediaKind(media.kind),
workspaceDir: normalizeOptionalText(media.workspaceDir),
...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}),
};
}
export function shouldPersistStructuredMediaEntries(
media: readonly PersistedUserTurnMediaInput[] | null | undefined,
): boolean {
return (media ?? []).some((entry) => {
const legacy = normalizeMediaEntryForTranscript(entry);
const structured = normalizeStructuredMediaEntryForTranscript(entry);
const structuredIdentity = structured.path ?? structured.url;
return (
structured.hydrationSuppressed === true ||
structuredIdentity !== legacy.path ||
Boolean(structured.url && structured.url !== legacy.path)
);
});
}
+21 -38
View File
@@ -1,18 +1,18 @@
// User turn transcript helpers extract user-turn text from session transcripts.
import path from "node:path";
import { mimeTypeFromFilePath } from "@openclaw/media-core/mime";
import type { AgentMessage } from "../../packages/agent-core/src/types.js";
import {
persistSessionTranscriptTurn,
type SessionTranscriptTurnPersistOptions,
} from "../config/sessions/session-accessor.js";
import {
projectMediaFacts,
resolveMediaFacts,
type MediaFact,
type MediaFactInput,
} from "../media/media-facts.js";
import { projectMediaFacts, resolveMediaFacts, type MediaFact } from "../media/media-facts.js";
import { applyInputProvenanceToUserMessage, normalizeInputProvenance } from "./input-provenance.js";
import {
normalizeMediaEntryForTranscript,
normalizeStructuredMediaEntryForTranscript,
resolveTranscriptMediaPath,
shouldPersistStructuredMediaEntries,
} from "./user-turn-transcript.media-normalize.js";
import type {
CreateUserTurnTranscriptRecorderParams,
PersistUserTurnTranscriptParams,
@@ -72,37 +72,6 @@ export function resolvePersistedUserTurnText(value: string | null | undefined):
return normalized;
}
function mediaTypeForTranscript(media: PersistedUserTurnMediaInput, mediaPath?: string): string {
return (
normalizeOptionalText(media.contentType) ??
normalizeOptionalText(media.kind) ??
mimeTypeFromFilePath(mediaPath) ??
"application/octet-stream"
);
}
function normalizeMediaEntryForTranscript(media: PersistedUserTurnMediaInput): MediaFactInput {
const rawPath = normalizeOptionalText(media.path) ?? normalizeOptionalText(media.url);
if (!rawPath) {
return {};
}
return {
path: resolveTranscriptMediaPath(rawPath, normalizeOptionalText(media.workspaceDir)),
contentType: mediaTypeForTranscript(media, rawPath),
};
}
const URL_LIKE_MEDIA_PATH_PATTERN = /^[a-z][a-z0-9+.-]*:/i;
function resolveTranscriptMediaPath(pathValue: string, workspaceDir: string | undefined): string {
// Relative staged media paths are anchored to the media workspace; absolute
// paths and URL-like refs are already stable transcript references.
if (!workspaceDir || path.isAbsolute(pathValue) || URL_LIKE_MEDIA_PATH_PATTERN.test(pathValue)) {
return pathValue;
}
return path.join(workspaceDir, pathValue);
}
function resolveTranscriptMediaType(params: {
explicitType: string | undefined;
mediaPath: string | undefined;
@@ -211,6 +180,7 @@ function readOpenClawMessageMeta(message: AgentMessage): Record<string, unknown>
export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedUserTurnMessage {
const mediaFields = buildPersistedUserTurnMediaFields(params.media);
const normalizedMedia = (params.media ?? []).map(normalizeStructuredMediaEntryForTranscript);
const text = normalizeTranscriptText(params.text);
// Storage is BARE (no timestamp prefix). The per-message timestamp is added
// at the single LLM-boundary stamping site (normalizeMessagesForLlmBoundary),
@@ -223,6 +193,19 @@ export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedU
...(params.senderIsOwner === undefined ? {} : { senderIsOwner: params.senderIsOwner }),
...senderMeta,
...(params.transport ? { transport: params.transport } : {}),
...(shouldPersistStructuredMediaEntries(params.media) ? { media: normalizedMedia } : {}),
...(params.mediaImageLayout
? {
mediaImageLayout: {
slots: params.mediaImageLayout.slots.map((slot) => ({ ...slot })),
...(params.mediaImageLayout.suppressedFactIndexes?.length
? {
suppressedFactIndexes: [...params.mediaImageLayout.suppressedFactIndexes],
}
: {}),
},
}
: {}),
};
const message = {
role: "user",
+12 -1
View File
@@ -14,7 +14,10 @@ type UserTurnSessionEntry = {
threadId?: string | number;
} & Record<string, unknown>;
export type PersistedUserTurnMediaInput = Pick<MediaFactInput, "contentType" | "path" | "url"> & {
export type PersistedUserTurnMediaInput = Pick<
MediaFactInput,
"contentType" | "hydrationSuppressed" | "path" | "url"
> & {
kind?: string | null;
workspaceDir?: string | null;
};
@@ -24,6 +27,14 @@ export type PersistedUserTurnMessage = Extract<AgentMessage, { role: "user" }>;
export type UserTurnInput = {
text?: string | null;
media?: readonly PersistedUserTurnMediaInput[] | null;
/** Restart-safe native image placement; model-visible prompt bytes remain separate. */
mediaImageLayout?: {
slots: readonly {
kind: "inline" | "offloaded";
factIndex?: number;
}[];
suppressedFactIndexes?: readonly number[];
} | null;
timestamp?: number;
idempotencyKey?: string;
senderIsOwner?: boolean;