mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-24 11:25:50 -06:00
refactor(media): hydrate and prune images from structured facts, delete marker-text parsing (#112913)
* refactor(media): hydrate and prune images from structured facts, delete marker-text parsing Embedded, CLI, host-context replay, and plugin-harness image hydration now consume the persisted/runtime MediaFact carrier instead of parsing OpenClaw-authored [media attached:]/[Image: source:] marker text. The prompt-text attachment parser family (splitPromptAndAttachmentRefs, leading/ trailing marker extractors, ref-count consumption, text-based prune scrubbing) is deleted; explicit user-authored paths and file URLs still hydrate. - Described-image suppression is a declared MediaFact.hydrationSuppressed field carried across copy/reprojection boundaries; the runtime symbol plumbing is removed. - Persisted facts plus mediaImageLayout are authoritative at every hydration entry point; image blocks reconcile by structural fact index, never byte/MIME equality. - Offloaded attachments (image and non-image) persist their media://inbound claim-check alias so ownership-aware history pruning can scrub markers after persistence; factless pre-MediaFact rows keep narrow legacy redaction. - History pruning removes facts, layout, and block provenance structurally before replay. Prompt bytes are golden-equal throughout; goldens from the fact-carrier PR are untouched. 1,303 tests across 44 files; delegated check:changed green (run 29980534186, pinned to the declared merge-base). * fix(media): fail missing inline image slots * test(agents): carry offloaded CLI retry image as a structured fact The retry-image reliability test modeled an offloaded attachment with only its [media attached:] marker text; offloaded attachments are carried as structured media facts and marker text is presentation-only, never parsed for hydration.
This commit is contained in:
committed by
GitHub
parent
b9a28cde42
commit
f6818ae5ea
@@ -3,11 +3,11 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "@openclaw/ai/internal/shared";
|
||||
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import type { ImageContent } from "openclaw/plugin-sdk/llm";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createSolidPngBuffer } from "../../test/helpers/image-fixtures.js";
|
||||
import { buildInboundMediaNoteProjection } from "../auto-reply/media-note.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js";
|
||||
import { escapeRegExp } from "../shared/regexp.js";
|
||||
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
|
||||
@@ -28,7 +28,7 @@ describe("prepareCliPromptImagePayload prompt references", () => {
|
||||
});
|
||||
|
||||
it("returns empty results when the prompt has no image refs", async () => {
|
||||
const loadImageFromRefSpy = vi.spyOn(promptImageUtils, "loadImageFromRef");
|
||||
const detectAndLoadPromptImagesSpy = vi.spyOn(promptImageUtils, "detectAndLoadPromptImages");
|
||||
const sanitizeImageBlocksSpy = vi.spyOn(toolImages, "sanitizeImageBlocks");
|
||||
|
||||
await expect(
|
||||
@@ -39,12 +39,12 @@ describe("prepareCliPromptImagePayload prompt references", () => {
|
||||
}),
|
||||
).resolves.toStrictEqual({ prompt: "just text" });
|
||||
|
||||
expect(loadImageFromRefSpy).not.toHaveBeenCalled();
|
||||
expect(detectAndLoadPromptImagesSpy).not.toHaveBeenCalled();
|
||||
expect(sanitizeImageBlocksSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not reload OpenClaw CLI image cache paths from prior prompt text", async () => {
|
||||
const loadImageFromRefSpy = vi.spyOn(promptImageUtils, "loadImageFromRef");
|
||||
const detectAndLoadPromptImagesSpy = vi.spyOn(promptImageUtils, "detectAndLoadPromptImages");
|
||||
const sanitizeImageBlocksSpy = vi.spyOn(toolImages, "sanitizeImageBlocks");
|
||||
|
||||
await expect(
|
||||
@@ -59,99 +59,115 @@ describe("prepareCliPromptImagePayload prompt references", () => {
|
||||
});
|
||||
|
||||
// Cached image paths are generated output, not fresh user references.
|
||||
expect(loadImageFromRefSpy).not.toHaveBeenCalled();
|
||||
expect(detectAndLoadPromptImagesSpy).not.toHaveBeenCalled();
|
||||
expect(sanitizeImageBlocksSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes the max-byte guardrail through load and sanitize", async () => {
|
||||
const loadedImage: ImageContent = {
|
||||
type: "image",
|
||||
data: "c29tZS1pbWFnZQ==",
|
||||
mimeType: "image/png",
|
||||
};
|
||||
const sanitizedImage: ImageContent = {
|
||||
type: "image",
|
||||
data: "c2FuaXRpemVkLWltYWdl",
|
||||
mimeType: "image/jpeg",
|
||||
};
|
||||
it("hydrates explicit prompt refs through the shared image loader", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(
|
||||
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-ref-image-"),
|
||||
);
|
||||
|
||||
const loadImageFromRefSpy = vi
|
||||
.spyOn(promptImageUtils, "loadImageFromRef")
|
||||
.mockResolvedValueOnce(loadedImage);
|
||||
const sanitizeImageBlocksSpy = vi
|
||||
.spyOn(toolImages, "sanitizeImageBlocks")
|
||||
.mockResolvedValueOnce({ images: [sanitizedImage], dropped: 0 });
|
||||
const imagePath = path.join(workspaceDir, "photo.png");
|
||||
const image = createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 });
|
||||
await fs.writeFile(imagePath, image);
|
||||
|
||||
try {
|
||||
const result = await prepareCliPromptImagePayload({
|
||||
backend: { command: "gemini", imagePathScope: "workspace" },
|
||||
prompt: "Look at /tmp/photo.png",
|
||||
prompt: `Look at ${imagePath}`,
|
||||
workspaceDir,
|
||||
});
|
||||
|
||||
const [ref, loadedWorkspaceDir, options] = loadImageFromRefSpy.mock.calls[0] ?? [];
|
||||
expect(ref?.resolved).toBe("/tmp/photo.png");
|
||||
expect(ref?.type).toBe("path");
|
||||
expect(loadedWorkspaceDir).toBe(workspaceDir);
|
||||
expect(options).toEqual({
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
workspaceOnly: undefined,
|
||||
sandbox: undefined,
|
||||
});
|
||||
expect(sanitizeImageBlocksSpy).toHaveBeenCalledWith([loadedImage], "prompt:images", {
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
});
|
||||
expect(result.imagePaths).toHaveLength(1);
|
||||
await expect(
|
||||
fs.readFile(expectDefined(result.imagePaths?.[0], "image path")),
|
||||
).resolves.toEqual(Buffer.from(sanitizedImage.data, "base64"));
|
||||
).resolves.toEqual(image);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("dedupes repeated refs and skips failed loads before sanitizing", async () => {
|
||||
const loadedImage: ImageContent = {
|
||||
type: "image",
|
||||
data: "b25lLWltYWdl",
|
||||
mimeType: "image/png",
|
||||
};
|
||||
|
||||
const loadImageFromRefSpy = vi
|
||||
.spyOn(promptImageUtils, "loadImageFromRef")
|
||||
.mockResolvedValueOnce(loadedImage)
|
||||
.mockResolvedValueOnce(null);
|
||||
const sanitizeImageBlocksSpy = vi
|
||||
.spyOn(toolImages, "sanitizeImageBlocks")
|
||||
.mockResolvedValueOnce({ images: [loadedImage], dropped: 0 });
|
||||
|
||||
const workspaceDir = await fs.mkdtemp(
|
||||
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-ref-dedupe-"),
|
||||
);
|
||||
const imagePath = path.join(workspaceDir, "a.png");
|
||||
await fs.writeFile(imagePath, createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 }));
|
||||
try {
|
||||
const result = await prepareCliPromptImagePayload({
|
||||
backend: { command: "gemini", imagePathScope: "workspace" },
|
||||
prompt: "Compare /tmp/a.png with /tmp/a.png and /tmp/b.png",
|
||||
prompt: `Compare ${imagePath} with ${imagePath} and ${path.join(workspaceDir, "missing.png")}`,
|
||||
workspaceDir,
|
||||
});
|
||||
|
||||
expect(loadImageFromRefSpy).toHaveBeenCalledTimes(2);
|
||||
expect(
|
||||
loadImageFromRefSpy.mock.calls.map(
|
||||
(call) => (call[0] as { resolved?: string } | undefined)?.resolved,
|
||||
),
|
||||
).toEqual(["/tmp/a.png", "/tmp/b.png"]);
|
||||
expect(sanitizeImageBlocksSpy).toHaveBeenCalledWith([loadedImage], "prompt:images", {
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
});
|
||||
expect(result.imagePaths).toHaveLength(1);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces structured image hydration failures", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(
|
||||
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-structured-failure-"),
|
||||
);
|
||||
try {
|
||||
await expect(
|
||||
prepareCliPromptImagePayload({
|
||||
backend: { command: "codex" },
|
||||
prompt: "describe the attachment",
|
||||
workspaceDir,
|
||||
media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }],
|
||||
}),
|
||||
).rejects.toThrow("failed to hydrate 1 structured image attachment");
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces inline sanitization failure when a preceding image fact is suppressed", async () => {
|
||||
await expect(
|
||||
prepareCliPromptImagePayload({
|
||||
backend: { command: "codex" },
|
||||
prompt: "already described",
|
||||
workspaceDir: "/tmp",
|
||||
images: [{ type: "image", data: "%%%", mimeType: "image/png" }],
|
||||
imageOrder: ["inline"],
|
||||
media: [
|
||||
{
|
||||
path: "/tmp/described-missing.png",
|
||||
contentType: "image/png",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
{ path: "/tmp/inline.png", contentType: "image/png" },
|
||||
],
|
||||
}),
|
||||
).rejects.toThrow("failed to hydrate 1 structured image attachment");
|
||||
});
|
||||
|
||||
it("accepts an intentionally non-hydrating remote-only image fact", async () => {
|
||||
const media = buildInboundMediaNoteProjection({
|
||||
MediaPaths: [""],
|
||||
MediaUrls: ["https://example.com/described.png"],
|
||||
MediaTypes: ["image/png"],
|
||||
MediaUnderstanding: [
|
||||
{
|
||||
kind: "image.description",
|
||||
attachmentIndex: 0,
|
||||
text: "already described",
|
||||
provider: "test",
|
||||
},
|
||||
],
|
||||
}).media;
|
||||
|
||||
await expect(
|
||||
prepareCliPromptImagePayload({
|
||||
backend: { command: "codex" },
|
||||
prompt: "already described",
|
||||
workspaceDir: "/tmp",
|
||||
media,
|
||||
}),
|
||||
).resolves.toEqual({ prompt: "already described" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildCliArgs", () => {
|
||||
@@ -366,6 +382,7 @@ describe("writeCliImages", () => {
|
||||
input: "arg",
|
||||
},
|
||||
prompt: `[media attached: ${sourceImage} (image/png)]\n\n<media:image>`,
|
||||
media: [{ path: sourceImage, contentType: "image/png" }],
|
||||
workspaceDir: tempDir,
|
||||
});
|
||||
const argv = buildCliArgs({
|
||||
@@ -411,6 +428,7 @@ describe("writeCliImages", () => {
|
||||
input: "stdin",
|
||||
},
|
||||
prompt,
|
||||
media: [{ path: sourceImage, contentType: "image/png" }],
|
||||
workspaceDir: tempDir,
|
||||
});
|
||||
const promptWithImages = prepared.prompt;
|
||||
@@ -560,6 +578,7 @@ describe("writeCliImages", () => {
|
||||
},
|
||||
],
|
||||
imageOrder: ["offloaded", "inline"],
|
||||
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
|
||||
});
|
||||
|
||||
expect(prepared.imagePaths).toHaveLength(2);
|
||||
|
||||
@@ -689,6 +689,9 @@ describe("runCliAgent reliability", () => {
|
||||
},
|
||||
],
|
||||
imageOrder: ["offloaded", "inline"],
|
||||
// Offloaded attachments are carried as structured facts; the trailing
|
||||
// marker text is presentation only and is never parsed for hydration.
|
||||
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
|
||||
};
|
||||
|
||||
const result = await runPreparedCliAgent(context);
|
||||
|
||||
@@ -593,9 +593,50 @@ describe("runCliAgent spawn path", () => {
|
||||
await expect(executePreparedCliRun(context)).rejects.toThrow(
|
||||
"paired-node Claude CLI sessions do not support attachments or images",
|
||||
);
|
||||
context.params.imagePrompt = undefined;
|
||||
context.params.media = [{ path: "/tmp/hydratable.png", kind: "image" }];
|
||||
await expect(executePreparedCliRun(context)).rejects.toThrow(
|
||||
"paired-node Claude CLI sessions do not support attachments or images",
|
||||
);
|
||||
expect(invokeNode).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows non-hydratable image facts on a text-only node turn", async () => {
|
||||
const invokeNode = vi.fn(async (params: Parameters<typeof invokeNodeClaudeCliRun>[0]) => {
|
||||
params.onProgress(
|
||||
[
|
||||
JSON.stringify({ type: "system", subtype: "init", session_id: "node-text-only" }),
|
||||
JSON.stringify({ type: "result", session_id: "node-text-only", result: "ok" }),
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
return {
|
||||
ok: true,
|
||||
payloadJSON: JSON.stringify({ exitCode: 0, stderrTail: "", truncated: false }),
|
||||
};
|
||||
});
|
||||
setCliRunnerExecuteTestDeps({ invokeNodeClaudeCliRun: invokeNode });
|
||||
const context = buildPreparedCliRunContext({
|
||||
provider: "claude-cli",
|
||||
model: "claude-opus-4-8",
|
||||
runId: "run-node-text-only-media-facts",
|
||||
prompt: "already described",
|
||||
sessionEntry: {
|
||||
sessionId: "openclaw-session",
|
||||
updatedAt: 1,
|
||||
execHost: "node",
|
||||
execNode: "node-a",
|
||||
},
|
||||
});
|
||||
context.params.media = [
|
||||
{ kind: "image" },
|
||||
{ kind: "image", url: "https://example.test/described.png" },
|
||||
];
|
||||
|
||||
await expect(executePreparedCliRun(context)).resolves.toMatchObject({ text: "ok" });
|
||||
expect(invokeNode).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("does not inject hardcoded 'Tools are disabled' text into CLI arguments", async () => {
|
||||
supervisorSpawnMock.mockResolvedValueOnce(
|
||||
createManagedRun({
|
||||
|
||||
@@ -68,6 +68,10 @@ import type {
|
||||
MessagingToolSend,
|
||||
MessagingToolSourceReplyPayload,
|
||||
} from "../embedded-agent-messaging.types.js";
|
||||
import {
|
||||
detectImageReferences,
|
||||
hasHydratableMediaImages,
|
||||
} from "../embedded-agent-runner/run/images.js";
|
||||
import {
|
||||
extractMessagingToolSendResult,
|
||||
extractMessagingToolSourceReplyPayload,
|
||||
@@ -458,7 +462,12 @@ export async function executePreparedCliRun(
|
||||
}),
|
||||
context.backendResolved.textTransforms?.input,
|
||||
);
|
||||
if (nodePlacement && ((params.images?.length ?? 0) > 0 || Boolean(params.imagePrompt?.trim()))) {
|
||||
if (
|
||||
nodePlacement &&
|
||||
((params.images?.length ?? 0) > 0 ||
|
||||
hasHydratableMediaImages(params.media) ||
|
||||
(params.imagePrompt ? detectImageReferences(params.imagePrompt).length > 0 : false))
|
||||
) {
|
||||
throw new Error("paired-node Claude CLI sessions do not support attachments or images");
|
||||
}
|
||||
const {
|
||||
@@ -474,6 +483,7 @@ export async function executePreparedCliRun(
|
||||
workspaceDir: context.workspaceDir,
|
||||
images: params.images,
|
||||
imageOrder: params.imageOrder,
|
||||
media: params.media,
|
||||
});
|
||||
prompt = promptWithImages;
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ import { privateFileStore } from "../../infra/private-file-store.js";
|
||||
import { tempWorkspace } from "../../infra/private-temp-workspace.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js";
|
||||
import type { ImageContent } from "../../llm/types.js";
|
||||
import type { MediaFact } from "../../media/media-facts.js";
|
||||
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
|
||||
import { KeyedAsyncQueue } from "../../plugin-sdk/keyed-async-queue.js";
|
||||
import type { CliBackendConfig } from "../../plugins/cli-backend.types.js";
|
||||
@@ -32,16 +33,13 @@ import type { EmbeddedContextFile } from "../embedded-agent-helpers.js";
|
||||
import {
|
||||
detectAndLoadPromptImages,
|
||||
detectImageReferences,
|
||||
loadImageFromRef,
|
||||
} from "../embedded-agent-runner/run/images.js";
|
||||
import { resolveDefaultModelForAgent } from "../model-selection.js";
|
||||
import type { AgentTool } from "../runtime/index.js";
|
||||
import type { SandboxFsBridge } from "../sandbox/fs-bridge.js";
|
||||
import { detectRuntimeShell } from "../shell-utils.js";
|
||||
import { buildConfiguredAgentSystemPrompt } from "../system-prompt-config.js";
|
||||
import { buildSystemPromptParams } from "../system-prompt-params.js";
|
||||
import type { SilentReplyPromptMode } from "../system-prompt.types.js";
|
||||
import { sanitizeImageBlocks } from "../tool-images.js";
|
||||
import { cliBackendLog } from "./log.js";
|
||||
import { formatTomlConfigOverride } from "./toml-inline.js";
|
||||
/** Re-export CLI reliability helpers used by older runner call sites. */
|
||||
@@ -362,44 +360,6 @@ function appendImagePathsToPrompt(prompt: string, paths: string[], prefix = ""):
|
||||
return `${trimmed}${separator}${paths.map((entry) => `${prefix}${entry}`).join("\n")}`;
|
||||
}
|
||||
|
||||
/** Loads and sanitizes image references found in prompt text. */
|
||||
async function loadPromptRefImages(params: {
|
||||
prompt: string;
|
||||
workspaceDir: string;
|
||||
maxBytes?: number;
|
||||
workspaceOnly?: boolean;
|
||||
sandbox?: { root: string; bridge: SandboxFsBridge };
|
||||
}): Promise<ImageContent[]> {
|
||||
const refs = detectImageReferences(params.prompt);
|
||||
if (refs.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const maxBytes = params.maxBytes ?? MAX_IMAGE_BYTES;
|
||||
const seen = new Set<string>();
|
||||
const images: ImageContent[] = [];
|
||||
for (const ref of refs) {
|
||||
const key = `${ref.type}:${ref.resolved}`;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
const image = await loadImageFromRef(ref, params.workspaceDir, {
|
||||
maxBytes,
|
||||
workspaceOnly: params.workspaceOnly,
|
||||
sandbox: params.sandbox,
|
||||
});
|
||||
if (image) {
|
||||
images.push(image);
|
||||
}
|
||||
}
|
||||
|
||||
const { images: sanitizedImages } = await sanitizeImageBlocks(images, "prompt:images", {
|
||||
maxBytes,
|
||||
});
|
||||
return sanitizedImages;
|
||||
}
|
||||
|
||||
/** Writes CLI image payloads to private paths and returns their file paths. */
|
||||
async function writeCliImages(params: {
|
||||
backend: CliBackendConfig;
|
||||
@@ -459,27 +419,35 @@ export async function prepareCliPromptImagePayload(params: {
|
||||
workspaceDir: string;
|
||||
images?: ImageContent[];
|
||||
imageOrder?: PromptImageOrderEntry[];
|
||||
media?: MediaFact[];
|
||||
}): Promise<{
|
||||
prompt: string;
|
||||
imagePaths?: string[];
|
||||
cleanupImages?: () => Promise<void>;
|
||||
}> {
|
||||
let prompt = params.prompt;
|
||||
const resolvedImages =
|
||||
params.imagePrompt !== undefined
|
||||
? (
|
||||
await detectAndLoadPromptImages({
|
||||
prompt: params.imagePrompt,
|
||||
workspaceDir: params.workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: params.images,
|
||||
imageOrder: params.imageOrder,
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
})
|
||||
).images
|
||||
: params.images && params.images.length > 0
|
||||
? params.images
|
||||
: await loadPromptRefImages({ prompt, workspaceDir: params.workspaceDir });
|
||||
const imagePrompt = params.imagePrompt ?? prompt;
|
||||
const needsHydration =
|
||||
params.imagePrompt !== undefined ||
|
||||
Boolean(params.media?.length) ||
|
||||
(!params.images?.length && detectImageReferences(imagePrompt).length > 0);
|
||||
const imageResult = needsHydration
|
||||
? await detectAndLoadPromptImages({
|
||||
prompt: imagePrompt,
|
||||
media: params.media,
|
||||
workspaceDir: params.workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: params.images,
|
||||
imageOrder: params.imageOrder,
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
})
|
||||
: undefined;
|
||||
if (imageResult?.failedMediaCount) {
|
||||
throw new Error(
|
||||
`failed to hydrate ${imageResult.failedMediaCount} structured image attachment(s) for CLI input`,
|
||||
);
|
||||
}
|
||||
const resolvedImages = imageResult?.images ?? params.images ?? [];
|
||||
if (resolvedImages.length === 0) {
|
||||
return { prompt };
|
||||
}
|
||||
|
||||
@@ -193,6 +193,7 @@ async function runEmbeddedAgentViaCliBackend(
|
||||
agentDir: params.agentDir,
|
||||
config: params.config,
|
||||
prompt: params.prompt,
|
||||
imagePrompt: params.prompt,
|
||||
media: params.media,
|
||||
provider: dispatch.provider,
|
||||
model: params.model,
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
/** Installs attempt-local context engine, tool-result, image, and frame guards. */
|
||||
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
|
||||
import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../../context-engine/host-compat.js";
|
||||
import { buildContextEngineRuntimeSettings } from "../../../context-engine/runtime-settings.js";
|
||||
import type { ContextEngine } from "../../../context-engine/types.js";
|
||||
import { isHeartbeatLifecycleRunKind } from "../../bootstrap-mode.js";
|
||||
import { DEFAULT_CONTEXT_TOKENS } from "../../defaults.js";
|
||||
import { resolveImageSanitizationLimits } from "../../image-sanitization.js";
|
||||
import type { SandboxContext } from "../../sandbox/types.js";
|
||||
import type { guardSessionManager } from "../../session-tool-result-guard-wrapper.js";
|
||||
import type { AgentSession } from "../../sessions/index.js";
|
||||
import { invalidateComputerFrameIfMissing } from "../../tools/computer-tool.js";
|
||||
@@ -29,6 +32,7 @@ export function installEmbeddedAttemptContextGuards(input: {
|
||||
attempt: EmbeddedRunAttemptParams;
|
||||
computerContextEpoch: { value: number };
|
||||
effectiveCwd: string;
|
||||
effectiveFsWorkspaceOnly: boolean;
|
||||
effectiveWorkspace: string;
|
||||
getPrePromptMessageCount: () => number;
|
||||
getPromptCache: () => EmbeddedRunAttemptResult["promptCache"];
|
||||
@@ -39,6 +43,7 @@ export function installEmbeddedAttemptContextGuards(input: {
|
||||
sessionAgentId: string;
|
||||
sessionManager: ReturnType<typeof guardSessionManager>;
|
||||
settingsManager: AgentSession["settingsManager"];
|
||||
sandbox?: SandboxContext | null;
|
||||
}): {
|
||||
getAfterTurnCheckpoint: () => number | null;
|
||||
remove: () => void;
|
||||
@@ -152,6 +157,17 @@ export function installEmbeddedAttemptContextGuards(input: {
|
||||
|
||||
const removeHistoryImagePruneContextTransform = installHistoryImagePruneContextTransform(
|
||||
activeSession.agent,
|
||||
{
|
||||
workspaceDir: input.effectiveWorkspace,
|
||||
model: attempt.model,
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
maxDimensionPx: resolveImageSanitizationLimits(attempt.config).maxDimensionPx,
|
||||
workspaceOnly: input.effectiveFsWorkspaceOnly,
|
||||
sandbox:
|
||||
input.sandbox?.enabled && input.sandbox.fsBridge
|
||||
? { root: input.sandbox.workspaceDir, bridge: input.sandbox.fsBridge }
|
||||
: undefined,
|
||||
},
|
||||
);
|
||||
const previousComputerFrameTransform = activeSession.agent.transformContext;
|
||||
activeSession.agent.transformContext = async (messages, signal) => {
|
||||
|
||||
@@ -65,7 +65,9 @@ describe("dispatchEmbeddedAttemptPrompt", () => {
|
||||
vi.clearAllMocks();
|
||||
hoisted.prepareEmbeddedAttemptPromptExecution.mockResolvedValue({
|
||||
images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
|
||||
imageFactIndexes: [null],
|
||||
detectedRefs: [],
|
||||
failedMediaCount: 0,
|
||||
loadedCount: 1,
|
||||
skippedCount: 0,
|
||||
});
|
||||
@@ -82,7 +84,9 @@ describe("dispatchEmbeddedAttemptPrompt", () => {
|
||||
order.push("images");
|
||||
return {
|
||||
images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
|
||||
imageFactIndexes: [null],
|
||||
detectedRefs: [],
|
||||
failedMediaCount: 0,
|
||||
loadedCount: 1,
|
||||
skippedCount: 0,
|
||||
};
|
||||
|
||||
@@ -12,7 +12,10 @@ const hoisted = vi.hoisted(() => ({
|
||||
withSessionWriteLock: vi.fn(async (operation: () => unknown) => await operation()),
|
||||
}));
|
||||
|
||||
vi.mock("@openclaw/media-core/constants", () => ({ MAX_IMAGE_BYTES: 1_234 }));
|
||||
vi.mock("@openclaw/media-core/constants", () => ({
|
||||
MAX_IMAGE_BYTES: 1_234,
|
||||
mediaKindFromMime: (mime: string) => (mime.startsWith("image/") ? "image" : "unknown"),
|
||||
}));
|
||||
vi.mock("../../image-sanitization.js", () => ({
|
||||
resolveImageSanitizationLimits: hoisted.resolveImageSanitizationLimits,
|
||||
}));
|
||||
@@ -70,7 +73,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
|
||||
hoisted.resolveImageSanitizationLimits.mockReturnValue({ maxDimensionPx: 2048 });
|
||||
hoisted.detectAndLoadPromptImages.mockResolvedValue({
|
||||
images: [{ type: "image", data: "loaded", mimeType: "image/png" }],
|
||||
imageFactIndexes: [null],
|
||||
detectedRefs: [],
|
||||
failedMediaCount: 0,
|
||||
loadedCount: 1,
|
||||
skippedCount: 0,
|
||||
});
|
||||
@@ -87,7 +92,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
|
||||
|
||||
expect(second).toEqual({
|
||||
images: [],
|
||||
imageFactIndexes: [],
|
||||
detectedRefs: [],
|
||||
failedMediaCount: 0,
|
||||
loadedCount: 0,
|
||||
skippedCount: 0,
|
||||
});
|
||||
@@ -136,7 +143,9 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
|
||||
});
|
||||
expect(result).toEqual({
|
||||
images: [{ type: "image", data: "loaded", mimeType: "image/png" }],
|
||||
imageFactIndexes: [null],
|
||||
detectedRefs: [],
|
||||
failedMediaCount: 0,
|
||||
loadedCount: 1,
|
||||
skippedCount: 0,
|
||||
});
|
||||
@@ -152,4 +161,81 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
|
||||
expect.objectContaining({ sandbox: undefined }),
|
||||
);
|
||||
});
|
||||
|
||||
it("reports failed hydration without consuming the embedded attempt fact", async () => {
|
||||
const base = createInput();
|
||||
const media = [{ path: "/tmp/missing.png", contentType: "image/png" }];
|
||||
const input = createInput({ attempt: { ...base.attempt, media } });
|
||||
hoisted.detectAndLoadPromptImages.mockResolvedValueOnce({
|
||||
images: [],
|
||||
imageFactIndexes: [],
|
||||
detectedRefs: [],
|
||||
failedMediaCount: 1,
|
||||
loadedCount: 0,
|
||||
skippedCount: 1,
|
||||
});
|
||||
|
||||
const result = await prepareEmbeddedAttemptPromptExecution(input);
|
||||
|
||||
expect(result.failedMediaCount).toBe(1);
|
||||
expect(input.attempt.media).toBe(media);
|
||||
});
|
||||
|
||||
it("uses persisted facts and layout as the current-turn provenance authority", async () => {
|
||||
const base = createInput();
|
||||
const persistedMessage = {
|
||||
role: "user" as const,
|
||||
content: "compare",
|
||||
MediaPaths: ["/tmp/inline.png", "/tmp/offloaded.png"],
|
||||
MediaTypes: ["image/png", "image/png"],
|
||||
__openclaw: {
|
||||
media: [
|
||||
{
|
||||
path: "/tmp/inline.png",
|
||||
contentType: "image/png",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
{ path: "/tmp/offloaded.png", contentType: "image/png" },
|
||||
],
|
||||
mediaImageLayout: {
|
||||
slots: [
|
||||
{ kind: "inline", factIndex: 0 },
|
||||
{ kind: "offloaded", factIndex: 1 },
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
const input = createInput({
|
||||
attempt: {
|
||||
...base.attempt,
|
||||
media: [{ path: "/tmp/offloaded.png", contentType: "image/png" }],
|
||||
userTurnTranscriptRecorder: {
|
||||
message: persistedMessage,
|
||||
resolveMessage: vi.fn(async () => persistedMessage),
|
||||
} as unknown as NonNullable<PromptExecutionInput["attempt"]["userTurnTranscriptRecorder"]>,
|
||||
},
|
||||
});
|
||||
|
||||
await prepareEmbeddedAttemptPromptExecution(input);
|
||||
|
||||
expect(hoisted.detectAndLoadPromptImages).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
media: [
|
||||
expect.objectContaining({
|
||||
path: "/tmp/inline.png",
|
||||
kind: "image",
|
||||
hydrationSuppressed: true,
|
||||
}),
|
||||
expect.objectContaining({ path: "/tmp/offloaded.png", kind: "image" }),
|
||||
],
|
||||
mediaImageLayout: {
|
||||
slots: [
|
||||
{ kind: "inline", factIndex: 0 },
|
||||
{ kind: "offloaded", factIndex: 1 },
|
||||
],
|
||||
suppressedFactIndexes: [],
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/** Prepares prompt-lock ownership and prompt-local images for submission. */
|
||||
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
|
||||
import type { OwnedSessionTranscriptCacheSnapshot } from "../../../config/sessions/transcript-write-context.js";
|
||||
import { resolveMediaFacts } from "../../../media/media-facts.js";
|
||||
import { resolveImageSanitizationLimits } from "../../image-sanitization.js";
|
||||
import type { SandboxContext } from "../../sandbox/types.js";
|
||||
import type { AgentSession } from "../../sessions/index.js";
|
||||
@@ -9,18 +10,31 @@ import {
|
||||
installPromptSubmissionLockRelease,
|
||||
} from "./attempt.session-lock.js";
|
||||
import { detectAndLoadPromptImages } from "./images.js";
|
||||
import {
|
||||
readPersistedMediaImageLayout,
|
||||
readPersistedPromptMediaFacts,
|
||||
} from "./prompt-image-metadata.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type PromptExecutionAttempt = Pick<
|
||||
EmbeddedRunAttemptParams,
|
||||
"config" | "imageOrder" | "images" | "model" | "sessionFile" | "sessionKey"
|
||||
| "config"
|
||||
| "imageOrder"
|
||||
| "images"
|
||||
| "media"
|
||||
| "model"
|
||||
| "sessionFile"
|
||||
| "sessionKey"
|
||||
| "userTurnTranscriptRecorder"
|
||||
>;
|
||||
type PromptImageResult = Awaited<ReturnType<typeof detectAndLoadPromptImages>>;
|
||||
|
||||
function emptyPromptImages(): PromptImageResult {
|
||||
return {
|
||||
images: [],
|
||||
imageFactIndexes: [],
|
||||
detectedRefs: [],
|
||||
failedMediaCount: 0,
|
||||
loadedCount: 0,
|
||||
skippedCount: 0,
|
||||
};
|
||||
@@ -57,12 +71,24 @@ export async function prepareEmbeddedAttemptPromptExecution(input: {
|
||||
input.sessionLockController.publishOwnedSessionFileSnapshot(snapshot),
|
||||
});
|
||||
|
||||
const persistedMessage =
|
||||
attempt.userTurnTranscriptRecorder?.message ??
|
||||
(await attempt.userTurnTranscriptRecorder?.resolveMessage());
|
||||
const persistedMedia = persistedMessage
|
||||
? (readPersistedPromptMediaFacts(persistedMessage) ??
|
||||
resolveMediaFacts(persistedMessage as unknown as Parameters<typeof resolveMediaFacts>[0]))
|
||||
: [];
|
||||
|
||||
return await detectAndLoadPromptImages({
|
||||
prompt: input.prompt,
|
||||
workspaceDir: input.effectiveWorkspace,
|
||||
model: attempt.model,
|
||||
existingImages: attempt.images,
|
||||
imageOrder: attempt.imageOrder,
|
||||
media: persistedMedia.length > 0 ? persistedMedia : attempt.media,
|
||||
mediaImageLayout: persistedMessage
|
||||
? readPersistedMediaImageLayout(persistedMessage)
|
||||
: undefined,
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
maxDimensionPx: resolveImageSanitizationLimits(attempt.config).maxDimensionPx,
|
||||
workspaceOnly: input.effectiveFsWorkspaceOnly,
|
||||
|
||||
@@ -39,6 +39,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
|
||||
activeContextEngine?: SessionManagerInput["activeContextEngine"];
|
||||
agentDir: string;
|
||||
effectiveCwd: string;
|
||||
effectiveFsWorkspaceOnly: boolean;
|
||||
effectiveWorkspace: string;
|
||||
initialSystemPrompt: string;
|
||||
isRawModelRun: boolean;
|
||||
@@ -167,6 +168,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
|
||||
attempt,
|
||||
computerContextEpoch: input.contextGuards.computerContextEpoch,
|
||||
effectiveCwd: input.effectiveCwd,
|
||||
effectiveFsWorkspaceOnly: input.effectiveFsWorkspaceOnly,
|
||||
effectiveWorkspace: input.effectiveWorkspace,
|
||||
getPrePromptMessageCount: () => state.prePromptMessageCount,
|
||||
getPromptCache: () => state.promptCache,
|
||||
@@ -177,6 +179,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
|
||||
sessionAgentId: input.sessionManager.sessionAgentId,
|
||||
sessionManager,
|
||||
settingsManager,
|
||||
sandbox: input.transport.sandbox,
|
||||
});
|
||||
input.lifecycle.onContextGuardsInstalled(contextGuards.remove);
|
||||
|
||||
|
||||
@@ -28,8 +28,59 @@ type PreparedProviderRuntimePluginHandle = ProviderRuntimePluginHandle & {
|
||||
prepared: true;
|
||||
};
|
||||
|
||||
export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptParams) {
|
||||
type AttemptWorkspaceParams = Pick<
|
||||
EmbeddedRunAttemptParams,
|
||||
| "agentId"
|
||||
| "config"
|
||||
| "cwd"
|
||||
| "execOverrides"
|
||||
| "sandboxSessionKey"
|
||||
| "sessionId"
|
||||
| "sessionKey"
|
||||
| "workspaceDir"
|
||||
>;
|
||||
|
||||
/** Resolves the shared workspace and sandbox policy used by native and plugin harnesses. */
|
||||
export async function resolveAttemptWorkspaceSandbox(params: AttemptWorkspaceParams) {
|
||||
const resolvedWorkspace = resolveUserPath(params.workspaceDir);
|
||||
await fs.mkdir(resolvedWorkspace, { recursive: true });
|
||||
const sandboxSessionKey =
|
||||
params.sandboxSessionKey?.trim() || params.sessionKey?.trim() || params.sessionId;
|
||||
const sandbox = await resolveSandboxContext({
|
||||
config: params.config,
|
||||
execOverrides: params.execOverrides,
|
||||
sessionKey: sandboxSessionKey,
|
||||
workspaceDir: resolvedWorkspace,
|
||||
});
|
||||
const effectiveWorkspace =
|
||||
sandbox?.enabled && sandbox.workspaceAccess !== "rw" ? sandbox.workspaceDir : resolvedWorkspace;
|
||||
const requestedCwd = params.cwd ? resolveUserPath(params.cwd) : undefined;
|
||||
if (sandbox?.enabled && requestedCwd && requestedCwd !== resolvedWorkspace) {
|
||||
throw new Error(
|
||||
"cwd override is not supported for sandboxed embedded agent runs; omit cwd or use the agent workspace as cwd",
|
||||
);
|
||||
}
|
||||
await fs.mkdir(effectiveWorkspace, { recursive: true });
|
||||
const { sessionAgentId } = resolveSessionAgentIds({
|
||||
sessionKey: params.sessionKey,
|
||||
config: params.config,
|
||||
agentId: params.agentId,
|
||||
});
|
||||
return {
|
||||
effectiveCwd: sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace),
|
||||
effectiveFsWorkspaceOnly: resolveAttemptFsWorkspaceOnly({
|
||||
config: params.config,
|
||||
sessionAgentId,
|
||||
}),
|
||||
effectiveWorkspace,
|
||||
resolvedWorkspace,
|
||||
sandbox,
|
||||
sandboxSessionKey,
|
||||
sessionAgentId,
|
||||
};
|
||||
}
|
||||
|
||||
export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptParams) {
|
||||
// Ultra is a logical orchestration mode, not a provider effort. Preserve it for
|
||||
// prompt/status surfaces, then lower only at agent-core and provider boundaries.
|
||||
const agentCoreThinkingLevel = mapThinkingLevel(params.thinkLevel);
|
||||
@@ -73,28 +124,8 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara
|
||||
}
|
||||
};
|
||||
|
||||
await fs.mkdir(resolvedWorkspace, { recursive: true });
|
||||
const sandboxSessionKey =
|
||||
params.sandboxSessionKey?.trim() || params.sessionKey?.trim() || params.sessionId;
|
||||
const sandbox = await resolveSandboxContext({
|
||||
config: params.config,
|
||||
execOverrides: params.execOverrides,
|
||||
sessionKey: sandboxSessionKey,
|
||||
workspaceDir: resolvedWorkspace,
|
||||
});
|
||||
const effectiveWorkspace = sandbox?.enabled
|
||||
? sandbox.workspaceAccess === "rw"
|
||||
? resolvedWorkspace
|
||||
: sandbox.workspaceDir
|
||||
: resolvedWorkspace;
|
||||
const requestedCwd = params.cwd ? resolveUserPath(params.cwd) : undefined;
|
||||
if (sandbox?.enabled && requestedCwd && requestedCwd !== resolvedWorkspace) {
|
||||
throw new Error(
|
||||
"cwd override is not supported for sandboxed embedded agent runs; omit cwd or use the agent workspace as cwd",
|
||||
);
|
||||
}
|
||||
const effectiveCwd = sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace);
|
||||
await fs.mkdir(effectiveWorkspace, { recursive: true });
|
||||
const workspace = await resolveAttemptWorkspaceSandbox(params);
|
||||
const { effectiveWorkspace } = workspace;
|
||||
|
||||
const getCurrentAttemptPluginMetadataSnapshot = (): PluginMetadataSnapshot | undefined =>
|
||||
params.preparedModelRuntime?.metadataSnapshot;
|
||||
@@ -141,22 +172,11 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara
|
||||
};
|
||||
return providerRuntimeHandle;
|
||||
};
|
||||
const { sessionAgentId } = resolveSessionAgentIds({
|
||||
sessionKey: params.sessionKey,
|
||||
config: params.config,
|
||||
agentId: params.agentId,
|
||||
});
|
||||
const effectiveFsWorkspaceOnly = resolveAttemptFsWorkspaceOnly({
|
||||
config: params.config,
|
||||
sessionAgentId,
|
||||
});
|
||||
prepStages.mark("workspace-sandbox");
|
||||
|
||||
return {
|
||||
agentCoreThinkingLevel,
|
||||
effectiveCwd,
|
||||
effectiveFsWorkspaceOnly,
|
||||
effectiveWorkspace,
|
||||
...workspace,
|
||||
emitCorePluginToolStageSummary,
|
||||
emitPrepStageSummary,
|
||||
getCurrentAttemptPluginMetadataSnapshot,
|
||||
@@ -164,9 +184,5 @@ export async function prepareEmbeddedAttemptSetup(params: EmbeddedRunAttemptPara
|
||||
prepStages,
|
||||
proactiveSubagentOrchestration,
|
||||
providerThinkingLevel,
|
||||
resolvedWorkspace,
|
||||
sandbox,
|
||||
sandboxSessionKey,
|
||||
sessionAgentId,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -60,14 +60,18 @@ describe("embedded OpenClaw queued steering cancellation", () => {
|
||||
{ path: "/tmp/a.png", contentType: "image/png" },
|
||||
{ path: "/tmp/b.pdf", contentType: "application/pdf" },
|
||||
];
|
||||
const imageOrder = ["offloaded", "inline"] as const;
|
||||
const activeSession: EmbeddedAgentActiveSessionSteerTarget = {
|
||||
steer,
|
||||
subscribe: () => () => {},
|
||||
};
|
||||
|
||||
await steerActiveSessionWithOptionalDeliveryWait(activeSession, "inspect both", { media });
|
||||
await steerActiveSessionWithOptionalDeliveryWait(activeSession, "inspect both", {
|
||||
media,
|
||||
imageOrder: [...imageOrder],
|
||||
});
|
||||
|
||||
expect(steer).toHaveBeenCalledWith("inspect both", undefined, undefined, media);
|
||||
expect(steer).toHaveBeenCalledWith("inspect both", undefined, undefined, media, imageOrder);
|
||||
});
|
||||
|
||||
it("waits for the queued user message_end transcript boundary", async () => {
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { toErrorObject } from "../../../infra/errors.js";
|
||||
import type { ImageContent } from "../../../llm/types.js";
|
||||
import type { MediaFact } from "../../../media/media-facts.js";
|
||||
import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js";
|
||||
import type { UserTurnTranscriptRecorder } from "../../../sessions/user-turn-transcript.types.js";
|
||||
import {
|
||||
cancelPendingAgentQuestionForSession,
|
||||
@@ -24,6 +25,7 @@ type EmbeddedAgentActiveSessionSteerTarget = {
|
||||
images?: ImageContent[],
|
||||
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder,
|
||||
media?: MediaFact[],
|
||||
imageOrder?: PromptImageOrderEntry[],
|
||||
): Promise<void>;
|
||||
subscribe(listener: (event: unknown) => void): () => void;
|
||||
};
|
||||
@@ -37,9 +39,10 @@ function steerActiveSession(
|
||||
images?: ImageContent[],
|
||||
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder,
|
||||
media?: MediaFact[],
|
||||
imageOrder?: PromptImageOrderEntry[],
|
||||
): Promise<void> {
|
||||
if (media?.length) {
|
||||
return activeSession.steer(text, images, userTurnTranscriptRecorder, media);
|
||||
return activeSession.steer(text, images, userTurnTranscriptRecorder, media, imageOrder);
|
||||
}
|
||||
return userTurnTranscriptRecorder
|
||||
? activeSession.steer(text, images, userTurnTranscriptRecorder)
|
||||
@@ -157,6 +160,7 @@ async function steerAndWaitForTranscriptCommit(
|
||||
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder,
|
||||
images?: ImageContent[],
|
||||
media?: MediaFact[],
|
||||
imageOrder?: PromptImageOrderEntry[],
|
||||
): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
@@ -243,6 +247,7 @@ async function steerAndWaitForTranscriptCommit(
|
||||
images,
|
||||
userTurnTranscriptRecorder,
|
||||
media,
|
||||
imageOrder,
|
||||
);
|
||||
steer.catch((err: unknown) => {
|
||||
finish(err);
|
||||
@@ -291,6 +296,7 @@ export async function steerActiveSessionWithOptionalDeliveryWait(
|
||||
options?.images,
|
||||
options?.userTurnTranscriptRecorder,
|
||||
options?.media,
|
||||
options?.imageOrder,
|
||||
);
|
||||
return;
|
||||
}
|
||||
@@ -301,5 +307,6 @@ export async function steerActiveSessionWithOptionalDeliveryWait(
|
||||
options.userTurnTranscriptRecorder,
|
||||
options.images,
|
||||
options.media,
|
||||
options.imageOrder,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -216,7 +216,9 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => {
|
||||
const prepareSessionManagerForRunMock = vi.fn(async (_params?: unknown) => undefined);
|
||||
const detectAndLoadPromptImagesMock = vi.fn(async () => ({
|
||||
images: [],
|
||||
imageFactIndexes: [],
|
||||
detectedRefs: [],
|
||||
failedMediaCount: 0,
|
||||
loadedCount: 0,
|
||||
skippedCount: 0,
|
||||
}));
|
||||
|
||||
@@ -330,6 +330,7 @@ export async function runEmbeddedAttempt(
|
||||
...(activeContextEngine ? { activeContextEngine } : {}),
|
||||
agentDir,
|
||||
effectiveCwd,
|
||||
effectiveFsWorkspaceOnly,
|
||||
effectiveWorkspace,
|
||||
initialSystemPrompt: preparedSystemPrompt.systemPromptText,
|
||||
isRawModelRun,
|
||||
|
||||
@@ -1,9 +1,18 @@
|
||||
// History image prune tests keep provider replay compact by replacing stale
|
||||
// image bytes and media references while preserving recent user context.
|
||||
// image bytes and fact-owned projections while preserving recent user context.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { AgentMessage } from "openclaw/plugin-sdk/agent-core";
|
||||
import type { ImageContent } from "openclaw/plugin-sdk/llm";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
attachRuntimePromptMediaFacts,
|
||||
readRuntimePromptMediaFacts,
|
||||
} from "../../../media/media-facts.js";
|
||||
import { buildPersistedUserTurnMessage } from "../../../sessions/user-turn-transcript.js";
|
||||
import { castAgentMessage } from "../../test-helpers/agent-message-fixtures.js";
|
||||
import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js";
|
||||
import {
|
||||
installHistoryImagePruneContextTransform,
|
||||
pruneProcessedHistoryImages,
|
||||
@@ -12,6 +21,8 @@ import {
|
||||
const PRUNED_HISTORY_IMAGE_MARKER = "[image data removed - already processed by model]";
|
||||
const PRUNED_HISTORY_MEDIA_REFERENCE_MARKER =
|
||||
"[media reference removed - already processed by model]";
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
|
||||
|
||||
function expectArrayMessageContent(
|
||||
message: AgentMessage | undefined,
|
||||
@@ -105,9 +116,23 @@ describe("pruneProcessedHistoryImages", () => {
|
||||
expect(content[0]?.type).toBe("text");
|
||||
});
|
||||
|
||||
it("scrubs old media attachment markers from text blocks", () => {
|
||||
// Text references are scrubbed alongside image blocks so old paths and
|
||||
// media URIs cannot rehydrate stale images on a later replay.
|
||||
it("strips explicit-image provenance when its old image block is pruned", () => {
|
||||
const message = castAgentMessage({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "explicit image" }, { ...image }],
|
||||
__openclaw: {
|
||||
mediaImageBlockFactIndexes: [null],
|
||||
mediaImageLayout: { slots: [{ kind: "inline" }] },
|
||||
},
|
||||
});
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const meta = (pruned[0] as unknown as Record<string, unknown>)["__openclaw"];
|
||||
|
||||
expect(meta).toBeUndefined();
|
||||
});
|
||||
|
||||
it("redacts factless legacy attachment text while pruning old image blocks", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
castAgentMessage({
|
||||
role: "user",
|
||||
@@ -147,7 +172,7 @@ describe("pruneProcessedHistoryImages", () => {
|
||||
expectContentBlock(originalContent[1], { type: "image", data: "abc" });
|
||||
});
|
||||
|
||||
it("scrubs old media attachment markers from string content without image blocks", () => {
|
||||
it("redacts factless legacy attachment text without image blocks", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
castAgentMessage({
|
||||
role: "user",
|
||||
@@ -157,16 +182,11 @@ describe("pruneProcessedHistoryImages", () => {
|
||||
];
|
||||
|
||||
const pruned = expectPrunedMessages(messages);
|
||||
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe(`please remember ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}`);
|
||||
const originalUser = messages[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(originalUser?.content).toBe(
|
||||
"please remember [media attached: media://inbound/stale-image.png]",
|
||||
);
|
||||
const user = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(user?.content).toBe(`please remember ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}`);
|
||||
});
|
||||
|
||||
it("prunes marked bare late-media turns identically to legacy literal turns", () => {
|
||||
it("prunes fact-owned and factless legacy late-media projections", () => {
|
||||
const fields = {
|
||||
role: "user" as const,
|
||||
MediaPath: "media://inbound/stale-image.png",
|
||||
@@ -178,8 +198,8 @@ describe("pruneProcessedHistoryImages", () => {
|
||||
__openclaw: { lateMedia: true },
|
||||
});
|
||||
const legacyString = castAgentMessage({
|
||||
...fields,
|
||||
content: "[media attached: media://inbound/stale-image.png]",
|
||||
role: "user",
|
||||
});
|
||||
const markedArray = castAgentMessage({
|
||||
...fields,
|
||||
@@ -187,7 +207,7 @@ describe("pruneProcessedHistoryImages", () => {
|
||||
__openclaw: { lateMedia: true },
|
||||
});
|
||||
const legacyArray = castAgentMessage({
|
||||
...fields,
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "[media attached: media://inbound/stale-image.png]" },
|
||||
{ ...image },
|
||||
@@ -199,12 +219,20 @@ describe("pruneProcessedHistoryImages", () => {
|
||||
const prunedMarkedArray = expectPrunedMessages([markedArray, ...oldEnoughTail()]);
|
||||
const prunedLegacyArray = expectPrunedMessages([legacyArray, ...oldEnoughTail()]);
|
||||
const markedStringOutput = prunedMarkedString as unknown as Array<{ content?: unknown }>;
|
||||
const legacyStringOutput = prunedLegacyString as unknown as Array<{ content?: unknown }>;
|
||||
const markedArrayOutput = prunedMarkedArray as unknown as Array<{ content?: unknown }>;
|
||||
const legacyStringOutput = prunedLegacyString as unknown as Array<{ content?: unknown }>;
|
||||
const legacyArrayOutput = prunedLegacyArray as unknown as Array<{ content?: unknown }>;
|
||||
|
||||
expect(markedStringOutput[0]?.content).toBe(legacyStringOutput[0]?.content);
|
||||
expect(markedArrayOutput[0]?.content).toEqual(legacyArrayOutput[0]?.content);
|
||||
expect(markedStringOutput[0]?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
expect(legacyStringOutput[0]?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
expect(markedArrayOutput[0]?.content).toEqual([
|
||||
{ type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER },
|
||||
{ type: "text", text: PRUNED_HISTORY_IMAGE_MARKER },
|
||||
]);
|
||||
expect(legacyArrayOutput[0]?.content).toEqual([
|
||||
{ type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER },
|
||||
{ type: "text", text: PRUNED_HISTORY_IMAGE_MARKER },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not replace a distinct caption on an old marked late-media turn", () => {
|
||||
@@ -216,12 +244,178 @@ describe("pruneProcessedHistoryImages", () => {
|
||||
__openclaw: { lateMedia: true },
|
||||
});
|
||||
|
||||
expect(pruneProcessedHistoryImages([message, ...oldEnoughTail()])).toBeNull();
|
||||
const output = [message] as unknown as Array<{ content?: unknown }>;
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const output = pruned as unknown as Array<{ content?: unknown; MediaPath?: unknown }>;
|
||||
expect(output[0]?.content).toBe("resolved subtitle");
|
||||
expect(output[0]?.MediaPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("scrubs bare old inbound media URIs from tool results", () => {
|
||||
it("drops runtime facts from captioned old turns without changing their text", () => {
|
||||
const message = attachRuntimePromptMediaFacts(
|
||||
castAgentMessage({ role: "user", content: "caption stays byte-identical" }),
|
||||
[{ path: "/tmp/stale.png", contentType: "image/png" }],
|
||||
);
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe("caption stays byte-identical");
|
||||
expect(firstUser && readRuntimePromptMediaFacts(firstUser)).toBeUndefined();
|
||||
expect(readRuntimePromptMediaFacts(message)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("redacts the exact fact-owned projection from a captioned old turn", () => {
|
||||
const imagePath = "/tmp/stale-owned.png";
|
||||
const message = attachRuntimePromptMediaFacts(
|
||||
castAgentMessage({
|
||||
role: "user",
|
||||
content: `[media attached: ${imagePath} (image/png)]\n\ncaption stays`,
|
||||
}),
|
||||
[{ path: imagePath, contentType: "image/png" }],
|
||||
);
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe(`${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}\n\ncaption stays`);
|
||||
});
|
||||
|
||||
it("redacts a persisted fact projection with a distinct URL", () => {
|
||||
const imagePath = "/tmp/stale-owned.png";
|
||||
const imageUrl = "https://example.test/stale-owned.png";
|
||||
const message = castAgentMessage({
|
||||
role: "user",
|
||||
content: `[media attached: ${imagePath} (image/png) | ${imageUrl}]`,
|
||||
__openclaw: {
|
||||
media: [
|
||||
{
|
||||
path: imagePath,
|
||||
url: imageUrl,
|
||||
contentType: "image/png",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
});
|
||||
|
||||
it("redacts a persisted relative projection after legacy path canonicalization", () => {
|
||||
const message = buildPersistedUserTurnMessage({
|
||||
text: "[media attached: ./old.png (image/png)]",
|
||||
media: [
|
||||
{
|
||||
path: "./old.png",
|
||||
workspaceDir: "/workspace",
|
||||
contentType: "image/png",
|
||||
},
|
||||
],
|
||||
}) as AgentMessage;
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
});
|
||||
|
||||
it("redacts a persisted relative projection without a dot prefix", () => {
|
||||
const message = buildPersistedUserTurnMessage({
|
||||
text: "[media attached: sub/old.png (image/png)]",
|
||||
media: [
|
||||
{
|
||||
path: "sub/old.png",
|
||||
workspaceDir: "/workspace",
|
||||
contentType: "image/png",
|
||||
},
|
||||
],
|
||||
}) as AgentMessage;
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe(PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
});
|
||||
|
||||
it("does not claim a basename-only marker for an absolute owned fact", () => {
|
||||
const message = buildPersistedUserTurnMessage({
|
||||
text: "caption mentions [media attached: ./photo.png (image/png)]",
|
||||
media: [{ path: "/workspace/sub/photo.png", contentType: "image/png" }],
|
||||
}) as AgentMessage;
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe("caption mentions [media attached: ./photo.png (image/png)]");
|
||||
});
|
||||
|
||||
it("preserves an unrelated legacy-shaped marker in a fact-backed caption", () => {
|
||||
const message = buildPersistedUserTurnMessage({
|
||||
text: "caption mentions [media attached: ./unrelated.png (image/png)]",
|
||||
media: [{ path: "./owned.png", workspaceDir: "/workspace", contentType: "image/png" }],
|
||||
}) as AgentMessage;
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe(
|
||||
"caption mentions [media attached: ./unrelated.png (image/png)]",
|
||||
);
|
||||
});
|
||||
|
||||
it("redacts separately projected fact lines in distinct text blocks", () => {
|
||||
const firstPath = "/tmp/first-owned.png";
|
||||
const secondPath = "/tmp/second-owned.png";
|
||||
const message = attachRuntimePromptMediaFacts(
|
||||
castAgentMessage({
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: `[media attached: ${firstPath} (image/png)]` },
|
||||
{ type: "text", text: `[media attached: ${secondPath} (image/png)]` },
|
||||
],
|
||||
}),
|
||||
[
|
||||
{ path: firstPath, contentType: "image/png" },
|
||||
{ path: secondPath, contentType: "image/png" },
|
||||
],
|
||||
);
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
expect(expectArrayMessageContent(pruned[0], "expected redacted text blocks")).toEqual([
|
||||
{ type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER },
|
||||
{ type: "text", text: PRUNED_HISTORY_MEDIA_REFERENCE_MARKER },
|
||||
]);
|
||||
});
|
||||
|
||||
it("redacts a fact-backed legacy image-source projection", () => {
|
||||
const imagePath = "/tmp/legacy-owned.jpg";
|
||||
const message = castAgentMessage({
|
||||
role: "user",
|
||||
content: `[Image: source: ${imagePath}]\ncaption stays`,
|
||||
MediaPath: imagePath,
|
||||
MediaPaths: [imagePath],
|
||||
MediaType: "image/jpeg",
|
||||
MediaTypes: ["image/jpeg"],
|
||||
});
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe(`${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER}\ncaption stays`);
|
||||
});
|
||||
|
||||
it("preserves a bare claim-check URI in a fact-backed caption", () => {
|
||||
const mediaRef = "media://inbound/captioned.png";
|
||||
const message = castAgentMessage({
|
||||
role: "user",
|
||||
content: `caption mentions ${mediaRef} as text`,
|
||||
MediaPath: mediaRef,
|
||||
MediaPaths: [mediaRef],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png"],
|
||||
});
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const firstUser = pruned[0] as Extract<AgentMessage, { role: "user" }> | undefined;
|
||||
expect(firstUser?.content).toBe(`caption mentions ${mediaRef} as text`);
|
||||
});
|
||||
|
||||
it("redacts bare old inbound media URIs from factless tool results", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
castAgentMessage({
|
||||
role: "toolResult",
|
||||
@@ -232,15 +426,8 @@ describe("pruneProcessedHistoryImages", () => {
|
||||
];
|
||||
|
||||
const pruned = expectPrunedMessages(messages);
|
||||
|
||||
const toolResult = pruned[0] as Extract<AgentMessage, { role: "toolResult" }> | undefined;
|
||||
expect(toolResult?.content).toBe(`previous ${PRUNED_HISTORY_MEDIA_REFERENCE_MARKER} result`);
|
||||
const originalToolResult = messages[0] as
|
||||
| Extract<AgentMessage, { role: "toolResult" }>
|
||||
| undefined;
|
||||
expect(originalToolResult?.content).toBe(
|
||||
"previous media://inbound/stale-screenshot.png result",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps image blocks that belong to the third-most-recent assistant turn", () => {
|
||||
@@ -453,4 +640,133 @@ describe("installHistoryImagePruneContextTransform", () => {
|
||||
restore();
|
||||
expect(agent.transformContext).toBe(originalTransformContext);
|
||||
});
|
||||
|
||||
it("does not legacy-redact a fact-backed caption on the wrapper second pass", async () => {
|
||||
const mediaRef = "media://inbound/captioned.png";
|
||||
const messages: AgentMessage[] = [
|
||||
castAgentMessage({
|
||||
role: "user",
|
||||
content: `caption mentions ${mediaRef} as text`,
|
||||
media: [{ url: mediaRef, contentType: "image/png" }],
|
||||
}),
|
||||
...oldEnoughTail(),
|
||||
];
|
||||
const agent = {
|
||||
transformContext: async (input: AgentMessage[]) =>
|
||||
input.map((message) => ({ ...message })) as AgentMessage[],
|
||||
};
|
||||
const restore = installHistoryImagePruneContextTransform(agent);
|
||||
|
||||
const replay = await agent.transformContext(messages);
|
||||
|
||||
expect((replay[0] as { content?: unknown }).content).toBe(
|
||||
`caption mentions ${mediaRef} as text`,
|
||||
);
|
||||
restore();
|
||||
});
|
||||
|
||||
it("hydrates recent facts before an existing transform clones messages", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-history-hydrate-"));
|
||||
const imagePath = path.join(workspaceDir, "photo.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const message = attachRuntimePromptMediaFacts(
|
||||
castAgentMessage({ role: "user", content: [{ type: "text", text: "describe" }] }),
|
||||
[{ path: imagePath, contentType: "image/png" }],
|
||||
);
|
||||
const agent = {
|
||||
transformContext: async (messages: AgentMessage[]) =>
|
||||
messages.map((entry) => {
|
||||
if (!("content" in entry)) {
|
||||
return { ...entry };
|
||||
}
|
||||
return {
|
||||
...entry,
|
||||
content: Array.isArray(entry.content)
|
||||
? entry.content.map((block: (typeof entry.content)[number]) => ({ ...block }))
|
||||
: entry.content,
|
||||
};
|
||||
}) as AgentMessage[],
|
||||
};
|
||||
const restore = installHistoryImagePruneContextTransform(agent, {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
try {
|
||||
const replay = await agent.transformContext([message]);
|
||||
expect(expectArrayMessageContent(replay[0], "expected hydrated content")).toEqual([
|
||||
{ type: "text", text: "describe" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
} finally {
|
||||
restore();
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("strips nested media metadata before old turns can rehydrate", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-pruned-nested-media-"));
|
||||
const imagePath = path.join(workspaceDir, "old.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const baseBridge = createHostSandboxFsBridge(workspaceDir);
|
||||
let hydrationReadCount = 0;
|
||||
const bridge = {
|
||||
...baseBridge,
|
||||
readFile: async (params: Parameters<typeof baseBridge.readFile>[0]) => {
|
||||
hydrationReadCount++;
|
||||
return await baseBridge.readFile(params);
|
||||
},
|
||||
};
|
||||
const message = castAgentMessage({
|
||||
role: "user",
|
||||
content: "[media attached: ./old.png (image/png)]",
|
||||
__openclaw: {
|
||||
media: [{ path: "./old.png", contentType: "image/png" }],
|
||||
mediaImageBlockFactIndexes: [0],
|
||||
mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] },
|
||||
},
|
||||
});
|
||||
const agent: {
|
||||
transformContext?: (messages: AgentMessage[]) => Promise<AgentMessage[]> | AgentMessage[];
|
||||
} = {};
|
||||
const restore = installHistoryImagePruneContextTransform(agent, {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
sandbox: { root: workspaceDir, bridge },
|
||||
});
|
||||
|
||||
try {
|
||||
const replay = await agent.transformContext?.([message, ...oldEnoughTail()]);
|
||||
const meta = (replay?.[0] as unknown as Record<string, unknown>)?.["__openclaw"] as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
expect(hydrationReadCount).toBe(0);
|
||||
expect(meta?.media).toBeUndefined();
|
||||
expect(meta?.mediaImageBlockFactIndexes).toBeUndefined();
|
||||
expect(meta?.mediaImageLayout).toBeUndefined();
|
||||
} finally {
|
||||
restore();
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("treats identity-less nested facts as structured ownership", () => {
|
||||
const message = castAgentMessage({
|
||||
role: "user",
|
||||
content: "[media attached: /tmp/unknown.png (image/png)]",
|
||||
__openclaw: {
|
||||
media: [{ kind: "image" }],
|
||||
mediaImageLayout: { slots: [], suppressedFactIndexes: [0] },
|
||||
},
|
||||
});
|
||||
|
||||
const pruned = expectPrunedMessages([message, ...oldEnoughTail()]);
|
||||
const first = pruned[0] as unknown as Record<string, unknown>;
|
||||
const meta = first["__openclaw"] as Record<string, unknown> | undefined;
|
||||
expect(first.content).toBe("[media attached: /tmp/unknown.png (image/png)]");
|
||||
expect(meta?.media).toBeUndefined();
|
||||
expect(meta?.mediaImageLayout).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,20 +1,31 @@
|
||||
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
|
||||
import {
|
||||
normalizeMediaFacts,
|
||||
projectMediaFacts,
|
||||
readRuntimePromptMediaFacts,
|
||||
resolveMediaFacts,
|
||||
type MediaFact,
|
||||
} from "../../../media/media-facts.js";
|
||||
/**
|
||||
* Prunes already-processed image payloads from replayed prompt history.
|
||||
*/
|
||||
import { buildLateMediaAttachedProjection } from "../../../sessions/user-turn-transcript.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
import { hasNonBlankUserText } from "./attempt.user-message-boundary.js";
|
||||
import { hydratePromptMediaMessages } from "./images.js";
|
||||
|
||||
/** Replacement text for old image blocks that were already available to the model. */
|
||||
const PRUNED_HISTORY_IMAGE_MARKER = "[image data removed - already processed by model]";
|
||||
|
||||
/** Replacement text for old textual media references that would otherwise be reloaded. */
|
||||
/** Replacement text for fact-owned late-media projections already processed by the model. */
|
||||
const PRUNED_HISTORY_MEDIA_REFERENCE_MARKER =
|
||||
"[media reference removed - already processed by model]";
|
||||
|
||||
const MEDIA_ATTACHED_HISTORY_REF_PATTERN = /\[media attached(?:\s+\d+\/\d+)?:\s*[^\]]+\]/gi;
|
||||
const MESSAGE_IMAGE_HISTORY_REF_PATTERN = /\[Image:\s*source:\s*[^\]]+\]/gi;
|
||||
const INBOUND_MEDIA_URI_HISTORY_REF_PATTERN = /\bmedia:\/\/inbound\/[^\]\s/\\]+/g;
|
||||
// Legacy replay hygiene only: factless pre-MediaFact rows past the prune cutoff
|
||||
// retain no attachment ownership. Fact-bearing messages never use these patterns.
|
||||
const LEGACY_MEDIA_ATTACHED_PATTERN = /\[media attached(?:\s+\d+\/\d+)?:\s*[^\]]+\]/gi;
|
||||
const LEGACY_IMAGE_SOURCE_PATTERN = /\[Image:\s*source:\s*[^\]]+\]/gi;
|
||||
const LEGACY_INBOUND_MEDIA_URI_PATTERN = /\bmedia:\/\/inbound\/[^\]\s/\\]+/g;
|
||||
|
||||
type PrunableContextAgent = {
|
||||
transformContext?: (
|
||||
@@ -29,6 +40,17 @@ type PrunableContextAgent = {
|
||||
* ones, so text-only turns consume the window.
|
||||
*/
|
||||
const PRESERVE_RECENT_COMPLETED_TURNS = 3;
|
||||
const PERSISTED_MEDIA_FIELD_KEYS = [
|
||||
"media",
|
||||
"MediaPath",
|
||||
"MediaPaths",
|
||||
"MediaUrl",
|
||||
"MediaUrls",
|
||||
"MediaType",
|
||||
"MediaTypes",
|
||||
"MediaTranscribedIndexes",
|
||||
"MediaWorkspaceDir",
|
||||
] as const;
|
||||
|
||||
function resolvePruneBeforeIndex(messages: AgentMessage[]): number {
|
||||
const completedTurnStarts: number[] = [];
|
||||
@@ -66,18 +88,133 @@ function resolvePruneBeforeIndex(messages: AgentMessage[]): number {
|
||||
return completedTurnStarts.at(-PRESERVE_RECENT_COMPLETED_TURNS) ?? -1;
|
||||
}
|
||||
|
||||
function pruneHistoryMediaReferenceText(text: string): string {
|
||||
function resolveMessageMediaFacts(message: AgentMessage): MediaFact[] {
|
||||
const runtimeMedia = readRuntimePromptMediaFacts(message);
|
||||
if (runtimeMedia) {
|
||||
return runtimeMedia;
|
||||
}
|
||||
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
|
||||
const nestedMedia =
|
||||
meta && typeof meta === "object" && !Array.isArray(meta)
|
||||
? (meta as Record<string, unknown>).media
|
||||
: undefined;
|
||||
return Array.isArray(nestedMedia)
|
||||
? normalizeMediaFacts(nestedMedia as MediaFact[])
|
||||
: resolveMediaFacts(message as unknown as Parameters<typeof resolveMediaFacts>[0]);
|
||||
}
|
||||
|
||||
function wasStructurallyMediaPruned(message: AgentMessage): boolean {
|
||||
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
|
||||
return (
|
||||
Boolean(meta) &&
|
||||
typeof meta === "object" &&
|
||||
!Array.isArray(meta) &&
|
||||
(meta as Record<string, unknown>).mediaImagePruned === true
|
||||
);
|
||||
}
|
||||
|
||||
function replaceLegacyFactlessMediaText(text: string): string {
|
||||
return text
|
||||
.replace(MEDIA_ATTACHED_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
|
||||
.replace(MESSAGE_IMAGE_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
|
||||
.replace(INBOUND_MEDIA_URI_HISTORY_REF_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
.replace(LEGACY_MEDIA_ATTACHED_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
|
||||
.replace(LEGACY_IMAGE_SOURCE_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
|
||||
.replace(LEGACY_INBOUND_MEDIA_URI_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
}
|
||||
|
||||
function factOwnsMarkerIdentity(identity: string, media: MediaFact[]): boolean {
|
||||
const normalizedIdentity = identity.replaceAll("\\", "/");
|
||||
return media.some((fact) =>
|
||||
[fact.path, fact.url].some((alias) => alias?.replaceAll("\\", "/") === normalizedIdentity),
|
||||
);
|
||||
}
|
||||
|
||||
function extractMediaAttachedIdentity(marker: string): string {
|
||||
const content = marker.replace(/^\[media attached(?:\s+\d+\/\d+)?:\s*/i, "").slice(0, -1);
|
||||
const mimeIndex = content.lastIndexOf(" (");
|
||||
const urlIndex = content.indexOf(" | ");
|
||||
const endIndexes = [mimeIndex, urlIndex].filter((index) => index >= 0);
|
||||
const endIndex = endIndexes.length > 0 ? Math.min(...endIndexes) : content.length;
|
||||
return content.slice(0, endIndex).trim();
|
||||
}
|
||||
|
||||
function replaceOwnedLegacyMediaMarkers(text: string, media: MediaFact[]): string {
|
||||
return text
|
||||
.replace(LEGACY_MEDIA_ATTACHED_PATTERN, (marker) =>
|
||||
factOwnsMarkerIdentity(extractMediaAttachedIdentity(marker), media)
|
||||
? PRUNED_HISTORY_MEDIA_REFERENCE_MARKER
|
||||
: marker,
|
||||
)
|
||||
.replace(LEGACY_IMAGE_SOURCE_PATTERN, (marker) => {
|
||||
const identity = marker
|
||||
.replace(/^\[Image:\s*source:\s*/i, "")
|
||||
.slice(0, -1)
|
||||
.trim();
|
||||
return factOwnsMarkerIdentity(identity, media)
|
||||
? PRUNED_HISTORY_MEDIA_REFERENCE_MARKER
|
||||
: marker;
|
||||
});
|
||||
}
|
||||
|
||||
function replaceOwnedMediaProjection(text: string, media: MediaFact[]): string {
|
||||
if (media.length === 0) {
|
||||
return text;
|
||||
}
|
||||
const projectionLines = new Set<string>();
|
||||
for (const facts of [media, ...media.map((fact) => [fact])]) {
|
||||
const projection = buildInboundMediaNoteProjection({
|
||||
media: facts,
|
||||
...projectMediaFacts(facts, "channel"),
|
||||
}).text;
|
||||
for (const line of projection?.split("\n") ?? []) {
|
||||
if (line) {
|
||||
projectionLines.add(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
let redacted = text;
|
||||
for (const line of projectionLines) {
|
||||
redacted = redacted.replaceAll(line, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
}
|
||||
for (const fact of media) {
|
||||
for (const alias of [fact.path, fact.url].filter((value): value is string => Boolean(value))) {
|
||||
redacted = redacted
|
||||
.replaceAll(`[Image: source: ${alias}]`, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
|
||||
.replaceAll(`[media attached: ${alias}]`, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
}
|
||||
}
|
||||
return replaceOwnedLegacyMediaMarkers(redacted, media);
|
||||
}
|
||||
|
||||
function cloneMessageWithContent(
|
||||
message: Extract<AgentMessage, { role: "user" | "toolResult" }>,
|
||||
content: typeof message.content,
|
||||
dropMedia = false,
|
||||
dropImageMetadata = dropMedia,
|
||||
): AgentMessage {
|
||||
return { ...message, content } as AgentMessage;
|
||||
const clone = { ...message, content } as AgentMessage & Record<string, unknown>;
|
||||
if (dropMedia) {
|
||||
for (const key of PERSISTED_MEDIA_FIELD_KEYS) {
|
||||
delete clone[key];
|
||||
}
|
||||
}
|
||||
if (dropImageMetadata) {
|
||||
const meta = clone["__openclaw"];
|
||||
const nextMeta =
|
||||
meta && typeof meta === "object" && !Array.isArray(meta)
|
||||
? { ...(meta as Record<string, unknown>) }
|
||||
: {};
|
||||
delete nextMeta.mediaImageBlockFactIndexes;
|
||||
delete nextMeta.mediaImageLayout;
|
||||
if (dropMedia) {
|
||||
delete nextMeta.media;
|
||||
nextMeta.mediaImagePruned = true;
|
||||
}
|
||||
if (Object.keys(nextMeta).length > 0) {
|
||||
clone["__openclaw"] = nextMeta;
|
||||
} else {
|
||||
delete clone["__openclaw"];
|
||||
}
|
||||
}
|
||||
return clone;
|
||||
}
|
||||
|
||||
/** Prunes old image payloads and references before later LLM-boundary synthesis. */
|
||||
@@ -93,12 +230,18 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess
|
||||
if (!message || (message.role !== "user" && message.role !== "toolResult")) {
|
||||
continue;
|
||||
}
|
||||
const media = message.role === "user" ? resolveMessageMediaFacts(message) : [];
|
||||
const hasOwnedMedia = media.length > 0;
|
||||
const structuredMediaWasPruned = wasStructurallyMediaPruned(message);
|
||||
|
||||
// Materialize blank marked turns here so this earlier boundary still prunes stale paths.
|
||||
const lateMediaText =
|
||||
const lateMediaProjection =
|
||||
message.role === "user" && !hasNonBlankUserText(message.content)
|
||||
? buildLateMediaAttachedProjection(message).text
|
||||
? buildLateMediaAttachedProjection(message)
|
||||
: undefined;
|
||||
const lateMediaText = lateMediaProjection?.media
|
||||
.map(() => PRUNED_HISTORY_MEDIA_REFERENCE_MARKER)
|
||||
.join("\n");
|
||||
const content = lateMediaText
|
||||
? Array.isArray(message.content)
|
||||
? ([{ type: "text", text: lateMediaText }, ...message.content] as typeof message.content)
|
||||
@@ -106,10 +249,14 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess
|
||||
: message.content;
|
||||
|
||||
if (typeof content === "string") {
|
||||
const prunedText = pruneHistoryMediaReferenceText(content);
|
||||
if (prunedText !== message.content) {
|
||||
const nextText = hasOwnedMedia
|
||||
? replaceOwnedMediaProjection(content, media)
|
||||
: structuredMediaWasPruned
|
||||
? content
|
||||
: replaceLegacyFactlessMediaText(content);
|
||||
if (nextText !== message.content || hasOwnedMedia) {
|
||||
prunedMessages ??= messages.slice();
|
||||
prunedMessages[i] = cloneMessageWithContent(message, prunedText);
|
||||
prunedMessages[i] = cloneMessageWithContent(message, nextText, hasOwnedMedia);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -121,16 +268,34 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess
|
||||
const nextContent = content.map((block) => {
|
||||
const typed = block as { type?: unknown; text?: unknown } | null | undefined;
|
||||
if (typed?.type === "text" && typeof typed.text === "string") {
|
||||
const text = pruneHistoryMediaReferenceText(typed.text);
|
||||
return text === typed.text ? block : ({ ...block, text } as (typeof content)[number]);
|
||||
const text = hasOwnedMedia
|
||||
? replaceOwnedMediaProjection(typed.text, media)
|
||||
: structuredMediaWasPruned
|
||||
? typed.text
|
||||
: replaceLegacyFactlessMediaText(typed.text);
|
||||
if (text !== typed.text) {
|
||||
return { ...block, text } as (typeof content)[number];
|
||||
}
|
||||
}
|
||||
return typed?.type === "image"
|
||||
? ({ type: "text", text: PRUNED_HISTORY_IMAGE_MARKER } as (typeof content)[number])
|
||||
: block;
|
||||
});
|
||||
if (lateMediaText || nextContent.some((block, index) => block !== content[index])) {
|
||||
const prunedImageBlock = content.some(
|
||||
(block) => (block as { type?: unknown } | null | undefined)?.type === "image",
|
||||
);
|
||||
if (
|
||||
hasOwnedMedia ||
|
||||
lateMediaText ||
|
||||
nextContent.some((block, index) => block !== content[index])
|
||||
) {
|
||||
prunedMessages ??= messages.slice();
|
||||
prunedMessages[i] = cloneMessageWithContent(message, nextContent);
|
||||
prunedMessages[i] = cloneMessageWithContent(
|
||||
message,
|
||||
nextContent,
|
||||
hasOwnedMedia,
|
||||
hasOwnedMedia || prunedImageBlock,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,13 +303,20 @@ export function pruneProcessedHistoryImages(messages: AgentMessage[]): AgentMess
|
||||
}
|
||||
|
||||
/** Installs an agent context transform that prunes old image/media history before model input. */
|
||||
export function installHistoryImagePruneContextTransform(agent: PrunableContextAgent): () => void {
|
||||
export function installHistoryImagePruneContextTransform(
|
||||
agent: PrunableContextAgent,
|
||||
mediaOptions?: Parameters<typeof hydratePromptMediaMessages>[1],
|
||||
): () => void {
|
||||
const originalTransformContext = agent.transformContext;
|
||||
agent.transformContext = async (messages: AgentMessage[], signal?: AbortSignal) => {
|
||||
const prunedInput = pruneProcessedHistoryImages(messages) ?? messages;
|
||||
const hydratedInput = mediaOptions
|
||||
? await hydratePromptMediaMessages(prunedInput, mediaOptions)
|
||||
: prunedInput;
|
||||
const transformed = originalTransformContext
|
||||
? await originalTransformContext.call(agent, messages, signal)
|
||||
: messages;
|
||||
const sourceMessages = Array.isArray(transformed) ? transformed : messages;
|
||||
? await originalTransformContext.call(agent, hydratedInput, signal)
|
||||
: hydratedInput;
|
||||
const sourceMessages = Array.isArray(transformed) ? transformed : hydratedInput;
|
||||
return pruneProcessedHistoryImages(sourceMessages) ?? sourceMessages;
|
||||
};
|
||||
return () => {
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createSolidPngBuffer } from "../../../../test/helpers/image-fixtures.js";
|
||||
import { detectAndLoadPromptImages } from "./images.js";
|
||||
|
||||
const HYDRATION_PARTS = ["inline", "offloaded", "suppressed", "explicit", "legacy"] as const;
|
||||
type HydrationPart = (typeof HYDRATION_PARTS)[number];
|
||||
type HydrationCombination = { name: string; parts: readonly HydrationPart[] };
|
||||
|
||||
function combinations<T>(values: readonly T[], size: number): T[][] {
|
||||
if (size === 0) {
|
||||
return [[]];
|
||||
}
|
||||
return values.flatMap((value, index) =>
|
||||
combinations(values.slice(index + 1), size - 1).map((tail) => [value].concat(tail)),
|
||||
);
|
||||
}
|
||||
|
||||
const HYDRATION_COMBINATIONS: HydrationCombination[] = [2, 3].flatMap((size) =>
|
||||
combinations(HYDRATION_PARTS, size).map((parts) => ({
|
||||
name: parts.join(" + "),
|
||||
parts,
|
||||
})),
|
||||
);
|
||||
|
||||
describe("hydration combination matrix", () => {
|
||||
it("attributes a suppressed-plus-inline sanitization failure to the inline fact", async () => {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "already described",
|
||||
media: [
|
||||
{
|
||||
path: "/tmp/described-missing.png",
|
||||
contentType: "image/png",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
{ path: "/tmp/inline.png", contentType: "image/png" },
|
||||
],
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }],
|
||||
imageOrder: ["inline"],
|
||||
});
|
||||
|
||||
expect(result.images).toEqual([]);
|
||||
expect(result.imageFactIndexes).toEqual([]);
|
||||
expect(result.loadedCount).toBe(0);
|
||||
expect(result.failedMediaCount).toBe(1);
|
||||
});
|
||||
|
||||
it.each(HYDRATION_COMBINATIONS)(
|
||||
"$name preserves materialization, order, and suppression invariants",
|
||||
async (testCase: HydrationCombination) => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-hydration-matrix-"));
|
||||
const has = (part: HydrationPart) => testCase.parts.includes(part);
|
||||
const inlineBuffer = createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 });
|
||||
const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 });
|
||||
const explicitBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 });
|
||||
const inlinePath = path.join(root, "inline.png");
|
||||
const offloadedPath = path.join(root, "offloaded.png");
|
||||
const explicitPath = path.join(root, "explicit.png");
|
||||
const suppressedPath = path.join(root, "suppressed-missing.png");
|
||||
if (has("offloaded")) {
|
||||
await fs.writeFile(offloadedPath, offloadedBuffer);
|
||||
}
|
||||
if (has("explicit")) {
|
||||
await fs.writeFile(explicitPath, explicitBuffer);
|
||||
}
|
||||
|
||||
const media: Array<{
|
||||
path: string;
|
||||
contentType: string;
|
||||
hydrationSuppressed?: boolean;
|
||||
}> = [];
|
||||
if (has("suppressed")) {
|
||||
media.push({
|
||||
path: suppressedPath,
|
||||
contentType: "image/png",
|
||||
hydrationSuppressed: true,
|
||||
});
|
||||
}
|
||||
const inlineFactIndex = has("inline") ? media.length : undefined;
|
||||
if (has("inline")) {
|
||||
media.push({ path: inlinePath, contentType: "image/png" });
|
||||
}
|
||||
const offloadedFactIndex = has("offloaded") ? media.length : undefined;
|
||||
if (has("offloaded")) {
|
||||
media.push({ path: offloadedPath, contentType: "image/png" });
|
||||
}
|
||||
|
||||
const inlineImage = {
|
||||
type: "image" as const,
|
||||
data: inlineBuffer.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
};
|
||||
const imageOrder = [
|
||||
...(has("inline") ? (["inline"] as const) : []),
|
||||
...(has("offloaded") ? (["offloaded"] as const) : []),
|
||||
];
|
||||
const existingImages = has("inline") ? [inlineImage] : undefined;
|
||||
const existingImageFactIndexes =
|
||||
has("legacy") && inlineFactIndex !== undefined ? [inlineFactIndex] : undefined;
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: has("explicit") ? `inspect ${explicitPath}` : "inspect attachments",
|
||||
media,
|
||||
workspaceDir: root,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages,
|
||||
existingImageFactIndexes,
|
||||
imageOrder: has("legacy") ? undefined : imageOrder,
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
const expectedImages = [
|
||||
...(has("inline") ? [inlineImage] : []),
|
||||
...(has("offloaded")
|
||||
? [
|
||||
{
|
||||
type: "image" as const,
|
||||
data: offloadedBuffer.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(has("explicit")
|
||||
? [
|
||||
{
|
||||
type: "image" as const,
|
||||
data: explicitBuffer.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
},
|
||||
]
|
||||
: []),
|
||||
];
|
||||
const expectedFactIndexes = [
|
||||
...(inlineFactIndex === undefined ? [] : [inlineFactIndex]),
|
||||
...(offloadedFactIndex === undefined ? [] : [offloadedFactIndex]),
|
||||
...(has("explicit") ? [null] : []),
|
||||
];
|
||||
const expectedLoadedCount = Number(has("offloaded")) + Number(has("explicit"));
|
||||
|
||||
expect(result.images).toEqual(expectedImages);
|
||||
expect(result.imageFactIndexes).toEqual(expectedFactIndexes);
|
||||
expect(result.loadedCount).toBe(expectedLoadedCount);
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.images).toHaveLength(
|
||||
Number(has("inline")) + Number(has("offloaded")) + Number(has("explicit")),
|
||||
);
|
||||
} finally {
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,293 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeMediaFacts, resolveMediaFacts } from "../../../media/media-facts.js";
|
||||
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
|
||||
import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js";
|
||||
import { detectAndLoadPromptImages, hasHydratableMediaImages } from "./images.js";
|
||||
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
|
||||
|
||||
describe("fact-carried image references", () => {
|
||||
it("counts only facts that will hydrate an image attachment", () => {
|
||||
expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", kind: "image" }])).toBe(true);
|
||||
// Legacy transcript projections persist bare kinds as the media type.
|
||||
expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", contentType: "image" }])).toBe(true);
|
||||
expect(hasHydratableMediaImages([{ path: "/tmp/anim.webp", contentType: "sticker" }])).toBe(
|
||||
true,
|
||||
);
|
||||
expect(hasHydratableMediaImages([{ kind: "image" }])).toBe(false);
|
||||
expect(
|
||||
hasHydratableMediaImages([{ url: "https://example.test/remote.png", kind: "image" }]),
|
||||
).toBe(false);
|
||||
expect(
|
||||
hasHydratableMediaImages([{ path: "https://example.test/remote.png", kind: "image" }]),
|
||||
).toBe(false);
|
||||
expect(hasHydratableMediaImages([])).toBe(false);
|
||||
expect(hasHydratableMediaImages(undefined)).toBe(false);
|
||||
for (const contentType of ["audio", "video", "document"]) {
|
||||
expect(hasHydratableMediaImages([{ path: "/tmp/photo.png", contentType }])).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("retains described-image suppression across fact copy boundaries", async () => {
|
||||
const normalized = normalizeMediaFacts([
|
||||
{
|
||||
path: "/tmp/described.png",
|
||||
contentType: "image/png",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
]);
|
||||
const reprojected = resolveMediaFacts({
|
||||
media: normalized,
|
||||
MediaPaths: ["/tmp/stale.png"],
|
||||
MediaTypes: ["application/octet-stream"],
|
||||
});
|
||||
|
||||
expect(reprojected).toEqual([
|
||||
expect.objectContaining({
|
||||
path: "/tmp/described.png",
|
||||
contentType: "image/png",
|
||||
kind: "image",
|
||||
hydrationSuppressed: true,
|
||||
}),
|
||||
]);
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "already described",
|
||||
media: reprojected,
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.images).toEqual([]);
|
||||
});
|
||||
|
||||
it("pairs identity-less facts with existing inline images when order metadata is absent", async () => {
|
||||
const existingImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "look",
|
||||
media: [{ kind: "image", contentType: "image/png" }],
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [existingImage],
|
||||
});
|
||||
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.images).toEqual([existingImage]);
|
||||
});
|
||||
|
||||
it("loads an explicit ref matching a fact sliced into an inline slot", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-inline-explicit-ref-"));
|
||||
const imagePath = path.join(workspaceDir, "photo.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: `compare ${imagePath}`,
|
||||
media: [{ path: imagePath, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [inlineImage],
|
||||
imageOrder: ["inline"],
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
expect(result.loadedCount).toBe(1);
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.images).toEqual([
|
||||
inlineImage,
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps identity-bearing refs when image order metadata has more inline slots", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-inline-order-"));
|
||||
const imagePath = path.join(workspaceDir, "offloaded.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const firstInline = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const secondInline = { ...firstInline };
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "compare",
|
||||
media: [{ path: imagePath, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [firstInline, secondInline],
|
||||
imageOrder: ["inline", "inline"],
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
expect(result.loadedCount).toBe(1);
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.images).toEqual([
|
||||
firstInline,
|
||||
secondInline,
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
expect(result.imageFactIndexes).toEqual([null, null, 0]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("fails an exact inline slot whose image block is missing", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-missing-inline-"));
|
||||
const imagePath = path.join(workspaceDir, "stale.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "inspect",
|
||||
media: [{ path: imagePath, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
imageOrder: ["inline"],
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
expect(result.loadedCount).toBe(0);
|
||||
expect(result.failedMediaCount).toBe(1);
|
||||
expect(result.images).toEqual([]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("hydrates a fact whose only local identity is a file URL", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-file-url-"));
|
||||
const imagePath = path.join(workspaceDir, "photo.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: [{ path: pathToFileURL(imagePath).href, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect(result.images).toEqual([
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("hydrates managed inbound media URIs before workspace path resolution", async () => {
|
||||
// Managed media URIs are canonical inbound attachment handles and should
|
||||
// work even when workspaceOnly would reject ordinary outside paths.
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-uri-"));
|
||||
const workspaceDir = path.join(stateDir, "workspace-agent");
|
||||
const inboundDir = path.join(stateDir, "media", "inbound");
|
||||
const mediaId = "telegram-photo.png";
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
await fs.mkdir(inboundDir, { recursive: true });
|
||||
await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
const image = result.images[0];
|
||||
|
||||
expect(image?.type).toBe("image");
|
||||
expect(image?.mimeType).toBe("image/png");
|
||||
expect(image?.data).toBe(TINY_PNG_BASE64);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("hydrates sandbox-staged inbound media URIs", async () => {
|
||||
const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-sbx-uri-"));
|
||||
const inboundDir = path.join(sandboxRoot, "media", "inbound");
|
||||
const mediaId = "telegram-photo.png";
|
||||
await fs.mkdir(inboundDir, { recursive: true });
|
||||
await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceDir: sandboxRoot,
|
||||
workspaceOnly: true,
|
||||
sandbox: {
|
||||
root: sandboxRoot,
|
||||
bridge: createHostSandboxFsBridge(sandboxRoot),
|
||||
},
|
||||
});
|
||||
const image = result.images[0];
|
||||
|
||||
expect(image?.type).toBe("image");
|
||||
expect(image?.mimeType).toBe("image/png");
|
||||
expect(image?.data).toBe(TINY_PNG_BASE64);
|
||||
} finally {
|
||||
await fs.rm(sandboxRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["traversal", "media://inbound/../secret.png"],
|
||||
["encoded traversal", "media://inbound/%2e%2e%2fsecret.png"],
|
||||
["null byte", "media://inbound/secret%00.png"],
|
||||
])("rejects %s claim-check facts", async (_label: string, mediaUrl: string) => {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "legacy ticket is carried structurally",
|
||||
media: [{ url: mediaUrl, contentType: "image/png" }],
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
|
||||
expect(result.loadedCount).toBe(0);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
expect(result.images).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("allows sandbox-validated host paths outside default media roots", async () => {
|
||||
const homeDir = os.homedir();
|
||||
await fs.mkdir(homeDir, { recursive: true });
|
||||
const sandboxParent = await fs.mkdtemp(path.join(homeDir, "openclaw-sandbox-image-"));
|
||||
try {
|
||||
const sandboxRoot = path.join(sandboxParent, "sandbox");
|
||||
await fs.mkdir(sandboxRoot, { recursive: true });
|
||||
const imagePath = path.join(sandboxRoot, "photo.png");
|
||||
const pngB64 = TINY_PNG_BASE64;
|
||||
await fs.writeFile(imagePath, Buffer.from(pngB64, "base64"));
|
||||
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: [{ path: "./photo.png", contentType: "image/png" }],
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceDir: sandboxRoot,
|
||||
sandbox: {
|
||||
root: sandboxRoot,
|
||||
bridge: createHostSandboxFsBridge(sandboxRoot),
|
||||
},
|
||||
});
|
||||
const image = result.images[0];
|
||||
|
||||
expect(image?.type).toBe("image");
|
||||
expect(image?.mimeType).toBe("image/png");
|
||||
expect(image?.data).toBe(TINY_PNG_BASE64);
|
||||
} finally {
|
||||
await fs.rm(sandboxParent, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
import { safeFileURLToPath } from "../../../infra/local-file-access.js";
|
||||
import {
|
||||
isImageMediaFact,
|
||||
normalizeMediaFacts,
|
||||
type MediaFact,
|
||||
} from "../../../media/media-facts.js";
|
||||
import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js";
|
||||
import { resolveUserPath } from "../../../utils.js";
|
||||
|
||||
const URL_SCHEME_PATTERN = /^[a-z][a-z0-9+.-]*:/i;
|
||||
const WINDOWS_DRIVE_PATH_PATTERN = /^[A-Za-z]:[\\/]/;
|
||||
|
||||
type DetectedImageRef = {
|
||||
raw: string;
|
||||
type: "path" | "media-uri";
|
||||
resolved: string;
|
||||
};
|
||||
|
||||
export type MediaImageRef = DetectedImageRef & {
|
||||
aliases: string[];
|
||||
detect?: boolean;
|
||||
factIndex: number;
|
||||
hydrate: boolean;
|
||||
workspaceDir?: string;
|
||||
};
|
||||
|
||||
export function isOpenClawCliImageCachePath(filePath: string): boolean {
|
||||
const parts = filePath.replaceAll("\\", "/").split("/");
|
||||
return parts.some((part, index) => {
|
||||
if (part === ".openclaw-cli-images") {
|
||||
return true;
|
||||
}
|
||||
const parent = parts[index - 1] ?? "";
|
||||
return part === "openclaw-cli-images" && /^openclaw(?:-\d+)?$/.test(parent);
|
||||
});
|
||||
}
|
||||
|
||||
function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef | undefined {
|
||||
if (!isImageMediaFact(fact)) {
|
||||
return undefined;
|
||||
}
|
||||
const mediaUri = [fact.url, fact.path].find((value) => value?.startsWith("media://inbound/"));
|
||||
const identity = mediaUri ?? fact.path ?? fact.url;
|
||||
if (!identity) {
|
||||
return fact.hydrationSuppressed === true
|
||||
? {
|
||||
aliases: [],
|
||||
detect: false,
|
||||
factIndex,
|
||||
raw: "",
|
||||
type: "path",
|
||||
resolved: "",
|
||||
hydrate: false,
|
||||
...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}),
|
||||
}
|
||||
: undefined;
|
||||
}
|
||||
let resolved = mediaUri;
|
||||
if (!resolved && identity && /^file:/i.test(identity)) {
|
||||
try {
|
||||
resolved = safeFileURLToPath(identity);
|
||||
} catch {
|
||||
resolved = undefined;
|
||||
}
|
||||
} else if (
|
||||
!resolved &&
|
||||
identity &&
|
||||
(!URL_SCHEME_PATTERN.test(identity) || WINDOWS_DRIVE_PATH_PATTERN.test(identity))
|
||||
) {
|
||||
resolved = identity;
|
||||
}
|
||||
if (resolved?.startsWith("~")) {
|
||||
resolved = resolveUserPath(resolved);
|
||||
}
|
||||
const hydrate = fact.hydrationSuppressed !== true;
|
||||
if (!resolved || isOpenClawCliImageCachePath(resolved)) {
|
||||
return {
|
||||
aliases: [fact.path, fact.url].filter((value): value is string => Boolean(value)),
|
||||
detect: false,
|
||||
factIndex,
|
||||
raw: identity,
|
||||
type: "path",
|
||||
resolved: identity,
|
||||
hydrate: false,
|
||||
...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}),
|
||||
};
|
||||
}
|
||||
return {
|
||||
aliases: [fact.path, fact.url, resolved].filter((value): value is string => Boolean(value)),
|
||||
factIndex,
|
||||
raw: mediaUri ?? fact.path ?? fact.url ?? resolved,
|
||||
type: mediaUri ? "media-uri" : "path",
|
||||
resolved,
|
||||
hydrate,
|
||||
...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function collectMediaImageRefs(
|
||||
media?: readonly MediaFact[],
|
||||
): Array<MediaImageRef | undefined> {
|
||||
return normalizeMediaFacts(media).flatMap((fact, factIndex) =>
|
||||
isImageMediaFact(fact) ? [mediaFactToImageRef(fact, factIndex)] : [],
|
||||
);
|
||||
}
|
||||
|
||||
export function collectIdentitylessMediaImageFactIndexes(media?: readonly MediaFact[]): number[] {
|
||||
return normalizeMediaFacts(media).flatMap((fact, factIndex) =>
|
||||
isImageMediaFact(fact) &&
|
||||
fact.hydrationSuppressed !== true &&
|
||||
fact.path === undefined &&
|
||||
fact.url === undefined
|
||||
? [factIndex]
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
// Guards for transports that cannot carry attachments (paired-node CLI): only
|
||||
// facts that will actually hydrate an image count; described/remote-only facts
|
||||
// whose hydration is suppressed must not block text-only prompts.
|
||||
export function hasHydratableMediaImages(media?: readonly MediaFact[]): boolean {
|
||||
return collectMediaImageRefs(media).some((ref) => ref?.hydrate === true);
|
||||
}
|
||||
|
||||
export function selectMediaImageRefs(params: {
|
||||
refs: Array<MediaImageRef | undefined>;
|
||||
existingImageCount: number;
|
||||
imageOrder?: readonly PromptImageOrderEntry[];
|
||||
}): Array<MediaImageRef | undefined> {
|
||||
const { refs } = params;
|
||||
if (!params.imageOrder?.length) {
|
||||
// Legacy turns (no layout metadata): identity-less facts are the inline
|
||||
// images' own slots — pair them positionally so they cannot count as failed
|
||||
// offloads; identity-bearing refs remain genuine offloaded attachments.
|
||||
let inlinePairs = params.existingImageCount;
|
||||
return refs.filter((ref) => {
|
||||
if (ref === undefined && inlinePairs > 0) {
|
||||
inlinePairs -= 1;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
if (refs.length !== params.imageOrder.length) {
|
||||
// Partial fact arrays cannot prove positional ownership. Keep every ref as
|
||||
// an offload so no attachment is silently consumed by an inline slot.
|
||||
return refs;
|
||||
}
|
||||
let remainingExisting = params.existingImageCount;
|
||||
return params.imageOrder.flatMap((entry, index) => {
|
||||
if (entry === "offloaded") {
|
||||
return [refs[index]];
|
||||
}
|
||||
if (remainingExisting > 0) {
|
||||
remainingExisting -= 1;
|
||||
return [];
|
||||
}
|
||||
return [undefined];
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,680 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createSolidPngBuffer } from "../../../../test/helpers/image-fixtures.js";
|
||||
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
|
||||
import {
|
||||
attachRuntimePromptMediaFacts,
|
||||
readRuntimePromptImageOrder,
|
||||
readRuntimePromptMediaFacts,
|
||||
} from "../../../media/media-facts.js";
|
||||
import {
|
||||
finalizeRuntimePromptImages,
|
||||
readRuntimePromptImageFactIndexes,
|
||||
} from "../../../media/runtime-prompt-image-provenance.js";
|
||||
import { buildPersistedUserTurnMessage } from "../../../sessions/user-turn-transcript.js";
|
||||
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
import {
|
||||
detectAndLoadPromptImages,
|
||||
detectImageReferences,
|
||||
hydratePromptMediaMessages,
|
||||
} from "./images.js";
|
||||
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
|
||||
|
||||
describe("structured prompt media replay", () => {
|
||||
it("keeps per-slot provenance when the same image object is reused", () => {
|
||||
const sharedImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const { images } = finalizeRuntimePromptImages([
|
||||
{ image: sharedImage, factIndex: 0 },
|
||||
{ image: sharedImage, factIndex: 1 },
|
||||
]);
|
||||
|
||||
expect(readRuntimePromptImageFactIndexes(images)).toEqual([0, 1]);
|
||||
});
|
||||
|
||||
it("retains the runtime fact carrier when queued hydration fails", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-runtime-failure-"));
|
||||
const media = [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }];
|
||||
const message = attachRuntimePromptMediaFacts(
|
||||
{ role: "user" as const, content: "missing attachment" },
|
||||
media,
|
||||
["offloaded"],
|
||||
) as unknown as AgentMessage;
|
||||
const runtimeMedia = readRuntimePromptMediaFacts(message);
|
||||
|
||||
try {
|
||||
const result = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect(readRuntimePromptMediaFacts(result[0] as AgentMessage)).toEqual(runtimeMedia);
|
||||
expect(readRuntimePromptImageOrder(result[0] as AgentMessage)).toEqual(["offloaded"]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not fail a described remote-only fact with no local identity", async () => {
|
||||
const media = buildInboundMediaNoteProjection({
|
||||
MediaPaths: [""],
|
||||
MediaUrls: ["https://example.com/described.png"],
|
||||
MediaTypes: ["image/png"],
|
||||
MediaUnderstanding: [
|
||||
{
|
||||
kind: "image.description",
|
||||
attachmentIndex: 0,
|
||||
text: "already described",
|
||||
provider: "test",
|
||||
},
|
||||
],
|
||||
}).media;
|
||||
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "already described",
|
||||
media,
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.detectedRefs).toEqual([]);
|
||||
expect(result.images).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports a fact-owned image dropped during sanitization", async () => {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "inspect it",
|
||||
media: [{ path: "/tmp/already-materialized.png", contentType: "image/png" }],
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }],
|
||||
existingImageFactIndexes: [0],
|
||||
imageOrder: ["inline"],
|
||||
});
|
||||
|
||||
expect(result.failedMediaCount).toBe(1);
|
||||
expect(result.images).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves persisted facts when replay hydration fails", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-failure-"));
|
||||
const missingPath = path.join(workspaceDir, "missing.png");
|
||||
const message = {
|
||||
role: "user" as const,
|
||||
content: "missing attachment",
|
||||
MediaPath: missingPath,
|
||||
MediaPaths: [missingPath],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png"],
|
||||
} as unknown as AgentMessage;
|
||||
|
||||
try {
|
||||
const result = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
const replayed = result[0] as unknown as Record<string, unknown>;
|
||||
expect(replayed.MediaPaths).toEqual([missingPath]);
|
||||
expect(replayed.content).toEqual([{ type: "text", text: "missing attachment" }]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("detects bracketed explicit paths but not textual claim-check tickets", () => {
|
||||
expect(detectImageReferences("inspect [source /tmp/photo.png]")).toEqual([
|
||||
{ raw: "/tmp/photo.png", type: "path", resolved: "/tmp/photo.png" },
|
||||
]);
|
||||
expect(detectImageReferences("[media attached: media://inbound/legacy.png]")).toEqual([]);
|
||||
expect(detectImageReferences("[media attached: /tmp/legacy.png (image/png)]")).toEqual([]);
|
||||
expect(detectImageReferences("[Image: source: /tmp/legacy.png]")).toEqual([]);
|
||||
});
|
||||
|
||||
it("dedupes the rendered alias of a private macOS path", async () => {
|
||||
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "[media attached: /var/tmp/aliased.png (image/png)]",
|
||||
media: [{ path: "/private/var/tmp/aliased.png", contentType: "image/png" }],
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
|
||||
expect(result.detectedRefs).toEqual([
|
||||
{
|
||||
raw: "/private/var/tmp/aliased.png",
|
||||
resolved: "/private/var/tmp/aliased.png",
|
||||
type: "path",
|
||||
},
|
||||
]);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
} finally {
|
||||
platformSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps private-var and var paths distinct away from Darwin", () => {
|
||||
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("linux");
|
||||
try {
|
||||
expect(
|
||||
detectImageReferences("compare /private/var/tmp/a.png and /var/tmp/a.png"),
|
||||
).toHaveLength(2);
|
||||
} finally {
|
||||
platformSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("expands a home-relative path carried by a media fact", async () => {
|
||||
const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-fact-home-"));
|
||||
const imagePath = path.join(homeDir, "Pictures", "photo.png");
|
||||
await fs.mkdir(path.dirname(imagePath), { recursive: true });
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const envSnapshot = captureEnv(["HOME"]);
|
||||
setTestEnvValue("HOME", homeDir);
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "describe it",
|
||||
media: [{ path: "~/Pictures/photo.png", contentType: "image/png" }],
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
localRoots: [homeDir],
|
||||
});
|
||||
expect(result.images).toHaveLength(1);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(homeDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps same-spelled relative refs from different workspaces distinct", async () => {
|
||||
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relative-roots-"));
|
||||
const stagedDir = path.join(rootDir, "staged");
|
||||
const currentDir = path.join(rootDir, "current");
|
||||
await fs.mkdir(stagedDir, { recursive: true });
|
||||
await fs.mkdir(currentDir, { recursive: true });
|
||||
await fs.writeFile(path.join(stagedDir, "photo.png"), Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
await fs.writeFile(
|
||||
path.join(currentDir, "photo.png"),
|
||||
createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 }),
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "compare ./photo.png",
|
||||
media: [{ path: "./photo.png", contentType: "image/png", workspaceDir: stagedDir }],
|
||||
workspaceDir: currentDir,
|
||||
model: { input: ["text", "image"] },
|
||||
localRoots: [stagedDir, currentDir],
|
||||
});
|
||||
expect(result.loadedCount).toBe(2);
|
||||
expect(result.images).toHaveLength(2);
|
||||
} finally {
|
||||
await fs.rm(rootDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not duplicate an already-materialized offloaded slot", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-materialized-offload-"));
|
||||
const imagePath = path.join(workspaceDir, "offloaded.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const image = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const persisted = buildPersistedUserTurnMessage({
|
||||
text: "already materialized",
|
||||
media: [{ path: imagePath, contentType: "image/png" }],
|
||||
mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] },
|
||||
}) as unknown as AgentMessage;
|
||||
|
||||
try {
|
||||
const first = await hydratePromptMediaMessages([persisted], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
const serialized = JSON.stringify(first[0]);
|
||||
const restored = JSON.parse(serialized) as AgentMessage;
|
||||
await fs.rm(imagePath);
|
||||
const replay = await hydratePromptMediaMessages([restored], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((replay[0] as unknown as { content?: unknown }).content).toEqual([
|
||||
{ type: "text", text: "already materialized" },
|
||||
image,
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("prefers persisted fact-index layout when runtime order is also present", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-layout-authority-"));
|
||||
const describedPath = path.join(workspaceDir, "described.png");
|
||||
const inlinePath = path.join(workspaceDir, "inline.png");
|
||||
const offloadedPath = path.join(workspaceDir, "offloaded.png");
|
||||
const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 });
|
||||
await fs.writeFile(describedPath, createSolidPngBuffer(1, 1, { r: 255, g: 0, b: 0 }));
|
||||
await fs.writeFile(inlinePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
await fs.writeFile(offloadedPath, offloadedBuffer);
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const media = [
|
||||
{ path: describedPath, contentType: "image/png" },
|
||||
{ path: inlinePath, contentType: "image/png" },
|
||||
{ path: offloadedPath, contentType: "image/png" },
|
||||
];
|
||||
const persisted = buildPersistedUserTurnMessage({
|
||||
text: "compare",
|
||||
media,
|
||||
mediaImageLayout: {
|
||||
slots: [
|
||||
{ kind: "inline", factIndex: 1 },
|
||||
{ kind: "offloaded", factIndex: 2 },
|
||||
],
|
||||
suppressedFactIndexes: [0],
|
||||
},
|
||||
});
|
||||
const message = attachRuntimePromptMediaFacts(
|
||||
{
|
||||
...persisted,
|
||||
content: [{ type: "text" as const, text: "compare" }, inlineImage],
|
||||
} as AgentMessage,
|
||||
media,
|
||||
["inline", "offloaded"],
|
||||
);
|
||||
|
||||
try {
|
||||
const first = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
const firstContent = (first[0] as unknown as { content?: unknown[] }).content;
|
||||
expect(firstContent).toEqual([
|
||||
{ type: "text", text: "compare" },
|
||||
inlineImage,
|
||||
{ type: "image", data: offloadedBuffer.toString("base64"), mimeType: "image/png" },
|
||||
]);
|
||||
|
||||
const serialized = JSON.stringify(first[0]);
|
||||
await fs.rm(describedPath);
|
||||
await fs.rm(inlinePath);
|
||||
await fs.rm(offloadedPath);
|
||||
const replay = await hydratePromptMediaMessages([JSON.parse(serialized) as AgentMessage], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((replay[0] as unknown as { content?: unknown[] }).content).toEqual(firstContent);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("retries an offloaded fact even when an unrelated explicit image exists", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-offload-retry-"));
|
||||
const explicitImage = {
|
||||
type: "image" as const,
|
||||
data: TINY_PNG_BASE64,
|
||||
mimeType: "image/png",
|
||||
};
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "retry missing attachment",
|
||||
media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }],
|
||||
mediaImageLayout: {
|
||||
slots: [{ kind: "offloaded", factIndex: 0 }],
|
||||
suppressedFactIndexes: [],
|
||||
},
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [explicitImage],
|
||||
existingImageFactIndexes: [null],
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect(result.failedMediaCount).toBe(1);
|
||||
expect(result.images).toEqual([explicitImage]);
|
||||
expect(result.imageFactIndexes).toEqual([null]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not assign a partial offloaded fact to an existing inline image", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-facts-"));
|
||||
const offloadedPath = path.join(workspaceDir, "offloaded.png");
|
||||
const offloadedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 });
|
||||
await fs.writeFile(offloadedPath, offloadedBuffer);
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "compare",
|
||||
media: [{ path: offloadedPath, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [inlineImage],
|
||||
imageOrder: ["inline", "offloaded"],
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.loadedCount).toBe(1);
|
||||
expect(result.images).toEqual([
|
||||
inlineImage,
|
||||
{ type: "image", data: offloadedBuffer.toString("base64"), mimeType: "image/png" },
|
||||
]);
|
||||
expect(result.imageFactIndexes).toEqual([null, 0]);
|
||||
expect(readRuntimePromptImageFactIndexes(result.images)).toEqual([null, 0]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves fact order for partial hydration without persisted layout", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-partial-order-"));
|
||||
const firstPath = path.join(workspaceDir, "first.png");
|
||||
const secondPath = path.join(workspaceDir, "second.png");
|
||||
const thirdPath = path.join(workspaceDir, "third.png");
|
||||
const secondBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 });
|
||||
const thirdBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 0, b: 255 });
|
||||
await fs.writeFile(thirdPath, thirdBuffer);
|
||||
const firstImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const secondImage = {
|
||||
type: "image" as const,
|
||||
data: secondBuffer.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "compare",
|
||||
media: [
|
||||
{ path: firstPath, contentType: "image/png" },
|
||||
{ path: secondPath, contentType: "image/png" },
|
||||
{ path: thirdPath, contentType: "image/png" },
|
||||
],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [firstImage, secondImage],
|
||||
existingImageFactIndexes: [0, 1],
|
||||
imageOrder: ["offloaded", "inline", "offloaded"],
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.images).toEqual([
|
||||
firstImage,
|
||||
secondImage,
|
||||
{ type: "image", data: thirdBuffer.toString("base64"), mimeType: "image/png" },
|
||||
]);
|
||||
expect(result.imageFactIndexes).toEqual([0, 1, 2]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("uses an unowned block for an inline slot when exact provenance is unavailable", async () => {
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const offloadedImage = {
|
||||
type: "image" as const,
|
||||
data: createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 }).toString("base64"),
|
||||
mimeType: "image/png",
|
||||
};
|
||||
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "compare",
|
||||
media: [{ kind: "document" }, { kind: "image" }, { kind: "image" }],
|
||||
mediaImageLayout: {
|
||||
slots: [
|
||||
{ kind: "inline", factIndex: 1 },
|
||||
{ kind: "offloaded", factIndex: 2 },
|
||||
],
|
||||
suppressedFactIndexes: [],
|
||||
},
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [inlineImage, offloadedImage],
|
||||
existingImageFactIndexes: [null, 2],
|
||||
});
|
||||
|
||||
expect(result.failedMediaCount).toBe(0);
|
||||
expect(result.images).toEqual([inlineImage, offloadedImage]);
|
||||
expect(result.imageFactIndexes).toEqual([null, 2]);
|
||||
});
|
||||
|
||||
it("uses explicit inline layout ownership for sanitization failures", async () => {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "inspect",
|
||||
media: [{ kind: "image" }, { kind: "image" }],
|
||||
mediaImageLayout: {
|
||||
slots: [{ kind: "inline", factIndex: 1 }],
|
||||
suppressedFactIndexes: [0],
|
||||
},
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [{ type: "image", data: "%%%", mimeType: "image/png" }],
|
||||
});
|
||||
|
||||
expect(result.failedMediaCount).toBe(1);
|
||||
expect(result.images).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports a missing unsuppressed inline layout slot", async () => {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "inspect",
|
||||
media: [{ kind: "image" }],
|
||||
mediaImageLayout: {
|
||||
slots: [{ kind: "inline", factIndex: 0 }],
|
||||
suppressedFactIndexes: [],
|
||||
},
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
|
||||
expect(result.failedMediaCount).toBe(1);
|
||||
expect(result.images).toEqual([]);
|
||||
});
|
||||
|
||||
it("pairs an identity-less legacy fact with its existing inline block", async () => {
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const message = {
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: "legacy identity-less" }, inlineImage],
|
||||
media: [{ kind: "image" }],
|
||||
} as unknown as AgentMessage;
|
||||
|
||||
const result = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
|
||||
expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([
|
||||
{ type: "text", text: "legacy identity-less" },
|
||||
inlineImage,
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not pair an unresolved remote legacy fact with an existing inline block", async () => {
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const message = {
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: "remote identity" }, inlineImage],
|
||||
media: [{ kind: "image", url: "https://example.test/remote.png" }],
|
||||
} as unknown as AgentMessage;
|
||||
|
||||
const result = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir: "/tmp",
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
const meta = (result[0] as unknown as Record<string, unknown>)["__openclaw"] as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
|
||||
expect(meta?.mediaImageBlockFactIndexes).toEqual([null]);
|
||||
expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([
|
||||
{ type: "text", text: "remote identity" },
|
||||
inlineImage,
|
||||
]);
|
||||
});
|
||||
|
||||
it("hydrates an identity-bearing legacy fact beside an unowned inline block", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-legacy-inline-"));
|
||||
const imagePath = path.join(workspaceDir, "inline.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const message = {
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: "legacy" }, inlineImage],
|
||||
MediaPath: imagePath,
|
||||
MediaPaths: [imagePath],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png"],
|
||||
} as unknown as AgentMessage;
|
||||
|
||||
try {
|
||||
const result = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
|
||||
{ type: "text", text: "legacy" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
inlineImage,
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps identity-bearing legacy facts ahead of an unowned inline block", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-legacy-order-"));
|
||||
const existingPath = path.join(workspaceDir, "existing.png");
|
||||
const hydratedPath = path.join(workspaceDir, "hydrated.png");
|
||||
const hydratedBuffer = createSolidPngBuffer(1, 1, { r: 0, g: 255, b: 0 });
|
||||
await fs.writeFile(existingPath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
await fs.writeFile(hydratedPath, hydratedBuffer);
|
||||
const existingImage = {
|
||||
type: "image" as const,
|
||||
data: TINY_PNG_BASE64,
|
||||
mimeType: "image/png",
|
||||
};
|
||||
const message = {
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: "legacy order" }, existingImage],
|
||||
MediaPath: existingPath,
|
||||
MediaPaths: [existingPath, hydratedPath],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png", "image/png"],
|
||||
} as unknown as AgentMessage;
|
||||
|
||||
try {
|
||||
const result = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((result[0] as unknown as { content?: unknown[] }).content).toEqual([
|
||||
{ type: "text", text: "legacy order" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
{ type: "image", data: hydratedBuffer.toString("base64"), mimeType: "image/png" },
|
||||
existingImage,
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("dedupes the local path alias of a claim-check fact", async () => {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-alias-"));
|
||||
const workspaceDir = path.join(stateDir, "workspace");
|
||||
const inboundDir = path.join(stateDir, "media", "inbound");
|
||||
const mediaId = "aliased.png";
|
||||
const imagePath = path.join(inboundDir, mediaId);
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
await fs.mkdir(inboundDir, { recursive: true });
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: `[media attached: ${imagePath} (image/png)]`,
|
||||
media: [{ path: imagePath, url: `media://inbound/${mediaId}`, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
|
||||
expect(result.loadedCount).toBe(1);
|
||||
expect(result.images).toHaveLength(1);
|
||||
expect(result.detectedRefs).toEqual([
|
||||
{
|
||||
raw: `media://inbound/${mediaId}`,
|
||||
resolved: `media://inbound/${mediaId}`,
|
||||
type: "media-uri",
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps described image facts suppressed across serialize and restore", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-suppressed-"));
|
||||
const imagePath = path.join(workspaceDir, "described.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const serialized = JSON.stringify(
|
||||
buildPersistedUserTurnMessage({
|
||||
text: "description already present",
|
||||
media: [
|
||||
{ path: imagePath, contentType: "image/png", hydrationSuppressed: true },
|
||||
{ kind: "sticker", hydrationSuppressed: true },
|
||||
],
|
||||
}),
|
||||
);
|
||||
const restored = JSON.parse(serialized) as AgentMessage;
|
||||
const meta = (restored as unknown as Record<string, unknown>)["__openclaw"] as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
const persistedMedia = meta?.media as
|
||||
| Array<{ path?: string; contentType?: string; hydrationSuppressed?: boolean }>
|
||||
| undefined;
|
||||
|
||||
try {
|
||||
expect(persistedMedia).toEqual([
|
||||
expect.objectContaining({
|
||||
path: imagePath,
|
||||
contentType: "image/png",
|
||||
hydrationSuppressed: true,
|
||||
}),
|
||||
expect.objectContaining({ kind: "sticker", hydrationSuppressed: true }),
|
||||
]);
|
||||
const replayHydration = await detectAndLoadPromptImages({
|
||||
prompt: "description already present",
|
||||
media: persistedMedia,
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect(replayHydration.failedMediaCount).toBe(0);
|
||||
expect(replayHydration.images).toEqual([]);
|
||||
|
||||
const result = await hydratePromptMediaMessages([restored], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
|
||||
{ type: "text", text: "description already present" },
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -5,11 +5,24 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "../../../infra/tmp-openclaw-dir.js";
|
||||
import {
|
||||
attachRuntimePromptMediaFacts,
|
||||
readRuntimePromptImageOrder,
|
||||
} from "../../../media/media-facts.js";
|
||||
import {
|
||||
buildPersistedUserTurnMessage,
|
||||
mergePreparedUserTurnMessageForRuntime,
|
||||
} from "../../../sessions/user-turn-transcript.js";
|
||||
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
|
||||
import { createHostSandboxFsBridge } from "../../test-helpers/host-sandbox-fs-bridge.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
import { createUnsafeMountedSandbox } from "../../test-helpers/unsafe-mounted-sandbox.js";
|
||||
import { detectAndLoadPromptImages, detectImageReferences, loadImageFromRef } from "./images.js";
|
||||
import {
|
||||
detectAndLoadPromptImages,
|
||||
detectImageReferences,
|
||||
hydratePromptMediaMessages,
|
||||
} from "./images.js";
|
||||
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
|
||||
@@ -154,10 +167,10 @@ describe("detectImageReferences", () => {
|
||||
});
|
||||
|
||||
it("does not leak parser state between calls", () => {
|
||||
expect(detectImageReferences("[media attached: /tmp/first.png (image/png)]")).toStrictEqual([
|
||||
expect(detectImageReferences("See /tmp/first.png")).toStrictEqual([
|
||||
{ raw: "/tmp/first.png", type: "path", resolved: "/tmp/first.png" },
|
||||
]);
|
||||
expect(detectImageReferences("[Image: source: /tmp/second.jpg]")).toStrictEqual([
|
||||
expect(detectImageReferences("See /tmp/second.jpg")).toStrictEqual([
|
||||
{ raw: "/tmp/second.jpg", type: "path", resolved: "/tmp/second.jpg" },
|
||||
]);
|
||||
const thirdPath = path.join(os.tmpdir(), "third.webp");
|
||||
@@ -265,67 +278,6 @@ describe("detectImageReferences", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("detects [Image: source: ...] format from messaging systems", () => {
|
||||
const ref = expectSingleImageReference(`What does this image show?
|
||||
[Image: source: /Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg]`);
|
||||
|
||||
expect(ref).toStrictEqual({
|
||||
raw: "/Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg",
|
||||
type: "path",
|
||||
resolved: "/Users/tyleryust/Library/Messages/Attachments/IMG_0043.jpeg",
|
||||
});
|
||||
});
|
||||
|
||||
it("handles complex message attachment paths", () => {
|
||||
const ref = expectSingleImageReference(
|
||||
"[Image: source: /Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg]",
|
||||
);
|
||||
|
||||
expect(ref).toStrictEqual({
|
||||
raw: "/Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg",
|
||||
type: "path",
|
||||
resolved:
|
||||
"/Users/tyleryust/Library/Messages/Attachments/23/03/AA4726EA-DB27-4269-BA56-1436936CC134/5E3E286A-F585-4E5E-9043-5BC2AFAFD81BIMG_0043.jpeg",
|
||||
});
|
||||
});
|
||||
|
||||
it("detects multiple images in [media attached: ...] format", () => {
|
||||
// Multi-file format uses separate brackets on separate lines
|
||||
const refs = expectImageReferenceCount(
|
||||
`[media attached: 2 files]
|
||||
[media attached 1/2: /Users/tyleryust/.openclaw/media/IMG_6430.jpeg (image/jpeg)]
|
||||
[media attached 2/2: /Users/tyleryust/.openclaw/media/IMG_6431.jpeg (image/jpeg)]
|
||||
what about these images?`,
|
||||
2,
|
||||
);
|
||||
|
||||
expect(refs).toStrictEqual([
|
||||
{
|
||||
raw: "/Users/tyleryust/.openclaw/media/IMG_6430.jpeg",
|
||||
type: "path",
|
||||
resolved: "/Users/tyleryust/.openclaw/media/IMG_6430.jpeg",
|
||||
},
|
||||
{
|
||||
raw: "/Users/tyleryust/.openclaw/media/IMG_6431.jpeg",
|
||||
type: "path",
|
||||
resolved: "/Users/tyleryust/.openclaw/media/IMG_6431.jpeg",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not double-count path and url in same bracket", () => {
|
||||
// Single file with URL (| separates path from url, not multiple files)
|
||||
const ref = expectSingleImageReference(
|
||||
"[media attached: /cache/IMG_6430.jpeg (image/jpeg) | /cache/IMG_6430.jpeg]",
|
||||
);
|
||||
|
||||
expect(ref).toStrictEqual({
|
||||
raw: "/cache/IMG_6430.jpeg",
|
||||
type: "path",
|
||||
resolved: "/cache/IMG_6430.jpeg",
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores remote URLs entirely (local-only)", () => {
|
||||
const refs = expectImageReferenceCount(
|
||||
`To send an image: MEDIA:https://example.com/image.jpg
|
||||
@@ -343,31 +295,6 @@ Also https://cdn.mysite.com/img.jpg`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("handles single file format with URL (no index)", () => {
|
||||
const ref =
|
||||
expectSingleImageReference(`[media attached: /cache/photo.jpeg (image/jpeg) | https://example.com/url]
|
||||
what is this?`);
|
||||
|
||||
expect(ref).toStrictEqual({
|
||||
raw: "/cache/photo.jpeg",
|
||||
type: "path",
|
||||
resolved: "/cache/photo.jpeg",
|
||||
});
|
||||
});
|
||||
|
||||
it("handles paths with spaces in filename", () => {
|
||||
// URL after | is https, not a local path, so only the local path should be detected
|
||||
const ref =
|
||||
expectSingleImageReference(`[media attached: /Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png (image/png) | https://example.com/same.png]
|
||||
what is this?`);
|
||||
|
||||
expect(ref).toStrictEqual({
|
||||
raw: "/Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png",
|
||||
type: "path",
|
||||
resolved: "/Users/test/.openclaw/media/ChatGPT Image Apr 21, 2025.png",
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores remote-host file URLs", () => {
|
||||
expectNoImageReferences("See file://attacker/share/evil.png");
|
||||
});
|
||||
@@ -385,107 +312,6 @@ what is this?`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("loadImageFromRef", () => {
|
||||
it("hydrates managed inbound media URIs before workspace path resolution", async () => {
|
||||
// Managed media URIs are canonical inbound attachment handles and should
|
||||
// work even when workspaceOnly would reject ordinary outside paths.
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-uri-"));
|
||||
const workspaceDir = path.join(stateDir, "workspace-agent");
|
||||
const inboundDir = path.join(stateDir, "media", "inbound");
|
||||
const mediaId = "telegram-photo.png";
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
await fs.mkdir(inboundDir, { recursive: true });
|
||||
await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
|
||||
try {
|
||||
const image = await loadImageFromRef(
|
||||
{
|
||||
raw: `media://inbound/${mediaId}`,
|
||||
type: "media-uri",
|
||||
resolved: `media://inbound/${mediaId}`,
|
||||
},
|
||||
workspaceDir,
|
||||
{ workspaceOnly: true },
|
||||
);
|
||||
|
||||
expect(image?.type).toBe("image");
|
||||
expect(image?.mimeType).toBe("image/png");
|
||||
expect(image?.data).toBe(TINY_PNG_BASE64);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("hydrates sandbox-staged inbound media URIs", async () => {
|
||||
const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-sbx-uri-"));
|
||||
const inboundDir = path.join(sandboxRoot, "media", "inbound");
|
||||
const mediaId = "telegram-photo.png";
|
||||
await fs.mkdir(inboundDir, { recursive: true });
|
||||
await fs.writeFile(path.join(inboundDir, mediaId), Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
try {
|
||||
const image = await loadImageFromRef(
|
||||
{
|
||||
raw: `media://inbound/${mediaId}`,
|
||||
type: "media-uri",
|
||||
resolved: `media://inbound/${mediaId}`,
|
||||
},
|
||||
sandboxRoot,
|
||||
{
|
||||
workspaceOnly: true,
|
||||
sandbox: {
|
||||
root: sandboxRoot,
|
||||
bridge: createHostSandboxFsBridge(sandboxRoot),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(image?.type).toBe("image");
|
||||
expect(image?.mimeType).toBe("image/png");
|
||||
expect(image?.data).toBe(TINY_PNG_BASE64);
|
||||
} finally {
|
||||
await fs.rm(sandboxRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("allows sandbox-validated host paths outside default media roots", async () => {
|
||||
const homeDir = os.homedir();
|
||||
await fs.mkdir(homeDir, { recursive: true });
|
||||
const sandboxParent = await fs.mkdtemp(path.join(homeDir, "openclaw-sandbox-image-"));
|
||||
try {
|
||||
const sandboxRoot = path.join(sandboxParent, "sandbox");
|
||||
await fs.mkdir(sandboxRoot, { recursive: true });
|
||||
const imagePath = path.join(sandboxRoot, "photo.png");
|
||||
const pngB64 = TINY_PNG_BASE64;
|
||||
await fs.writeFile(imagePath, Buffer.from(pngB64, "base64"));
|
||||
|
||||
const image = await loadImageFromRef(
|
||||
{
|
||||
raw: "./photo.png",
|
||||
type: "path",
|
||||
resolved: "./photo.png",
|
||||
},
|
||||
sandboxRoot,
|
||||
{
|
||||
sandbox: {
|
||||
root: sandboxRoot,
|
||||
bridge: createHostSandboxFsBridge(sandboxRoot),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(image?.type).toBe("image");
|
||||
expect(image?.mimeType).toBe("image/png");
|
||||
expect(image?.data).toBe(TINY_PNG_BASE64);
|
||||
} finally {
|
||||
await fs.rm(sandboxParent, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectAndLoadPromptImages", () => {
|
||||
it("returns no images for non-vision models even when existing images are provided", async () => {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
@@ -529,6 +355,7 @@ describe("detectAndLoadPromptImages", () => {
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "[media attached: ./photo.png (image/png)]\ndescribe it",
|
||||
media: [{ path: imagePath, contentType: "image/png" }],
|
||||
workspaceDir: stateDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [{ type: "image", data: pngB64, mimeType: "image/png" }],
|
||||
@@ -536,7 +363,7 @@ describe("detectAndLoadPromptImages", () => {
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
expect(result.detectedRefs).toHaveLength(1);
|
||||
expect(result.detectedRefs).toEqual([{ raw: imagePath, type: "path", resolved: imagePath }]);
|
||||
expect(result.loadedCount).toBe(0);
|
||||
expect(result.skippedCount).toBe(0);
|
||||
expect(result.images).toEqual([{ type: "image", data: pngB64, mimeType: "image/png" }]);
|
||||
@@ -545,6 +372,65 @@ describe("detectAndLoadPromptImages", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("uses a described fact identity to suppress its generated media-note path", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-described-dedupe-"));
|
||||
const imagePath = path.join(workspaceDir, "photo.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: `[media attached: ${imagePath} (image/png)]`,
|
||||
media: buildInboundMediaNoteProjection({
|
||||
MediaPath: imagePath,
|
||||
MediaType: "image/png",
|
||||
MediaUnderstanding: [
|
||||
{
|
||||
kind: "image.description",
|
||||
attachmentIndex: 0,
|
||||
text: "already described",
|
||||
provider: "test",
|
||||
},
|
||||
],
|
||||
}).media,
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
|
||||
expect(result.detectedRefs).toEqual([{ raw: imagePath, type: "path", resolved: imagePath }]);
|
||||
expect(result.loadedCount).toBe(0);
|
||||
expect(result.images).toEqual([]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("dedupes a relative fact projection against the fact workspace", async () => {
|
||||
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-fact-workspace-dedupe-"));
|
||||
const stagedDir = path.join(rootDir, "staged");
|
||||
const currentDir = path.join(rootDir, "current");
|
||||
await fs.mkdir(stagedDir, { recursive: true });
|
||||
await fs.mkdir(currentDir, { recursive: true });
|
||||
await fs.writeFile(path.join(stagedDir, "photo.png"), Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
await fs.writeFile(path.join(currentDir, "photo.png"), TINY_GIF_BUFFER);
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "[media attached: ./photo.png (image/png)]",
|
||||
media: [{ path: "./photo.png", contentType: "image/png", workspaceDir: stagedDir }],
|
||||
workspaceDir: currentDir,
|
||||
model: { input: ["text", "image"] },
|
||||
localRoots: [stagedDir, currentDir],
|
||||
});
|
||||
|
||||
expect(result.loadedCount).toBe(1);
|
||||
expect(result.images).toEqual([
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(rootDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps distinct inline attachments with identical bytes", async () => {
|
||||
const pngB64 = TINY_PNG_BASE64;
|
||||
const image = { type: "image" as const, data: pngB64, mimeType: "image/png" };
|
||||
@@ -561,6 +447,37 @@ describe("detectAndLoadPromptImages", () => {
|
||||
expect(result.images).toEqual([image, image]);
|
||||
});
|
||||
|
||||
it("keeps offloaded-only facts when existing images have no order metadata", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-unordered-images-"));
|
||||
const imagePath = path.join(workspaceDir, "offloaded.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const inlineImage = {
|
||||
type: "image" as const,
|
||||
data: TINY_GIF_BUFFER.toString("base64"),
|
||||
mimeType: "image/gif",
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "compare",
|
||||
media: [{ path: imagePath, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [inlineImage],
|
||||
imageOrder: [],
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
expect(result.loadedCount).toBe(1);
|
||||
expect(result.images).toEqual([
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
inlineImage,
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("classifies prompt and attachment refs while preserving mixed attachment order", async () => {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-order-"));
|
||||
const inboundDir = path.join(stateDir, "media", "inbound");
|
||||
@@ -579,6 +496,7 @@ describe("detectAndLoadPromptImages", () => {
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt,
|
||||
media: [{ url: "media://inbound/att-b.gif", contentType: "image/gif" }],
|
||||
workspaceDir: stateDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }],
|
||||
@@ -587,11 +505,6 @@ describe("detectAndLoadPromptImages", () => {
|
||||
});
|
||||
|
||||
expect(result.detectedRefs).toEqual([
|
||||
{
|
||||
raw: "media://inbound/prompt-ref.png",
|
||||
type: "media-uri",
|
||||
resolved: "media://inbound/prompt-ref.png",
|
||||
},
|
||||
{
|
||||
raw: "media://inbound/att-b.gif",
|
||||
type: "media-uri",
|
||||
@@ -599,12 +512,11 @@ describe("detectAndLoadPromptImages", () => {
|
||||
},
|
||||
{ raw: "./prompt-b.png", type: "path", resolved: "./prompt-b.png" },
|
||||
]);
|
||||
expect(result.loadedCount).toBe(3);
|
||||
expect(result.loadedCount).toBe(2);
|
||||
expect(result.images).toEqual([
|
||||
{ type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
@@ -612,6 +524,63 @@ describe("detectAndLoadPromptImages", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves an empty described-image slot in mixed attachment order", async () => {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-described-image-order-"));
|
||||
const inboundDir = path.join(stateDir, "media", "inbound");
|
||||
const mediaId = "remaining.gif";
|
||||
await fs.mkdir(inboundDir, { recursive: true });
|
||||
await fs.writeFile(path.join(inboundDir, mediaId), TINY_GIF_BUFFER);
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "compare",
|
||||
media: [
|
||||
{ kind: "image" },
|
||||
{ kind: "image" },
|
||||
{ url: `media://inbound/${mediaId}`, contentType: "image/gif" },
|
||||
],
|
||||
workspaceDir: stateDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: [inlineImage],
|
||||
imageOrder: ["offloaded", "inline", "offloaded"],
|
||||
});
|
||||
|
||||
expect(result.images).toEqual([
|
||||
inlineImage,
|
||||
{ type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" },
|
||||
]);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not load an explicit prompt ref twice when the same fact owns it", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-fact-"));
|
||||
await fs.writeFile(path.join(workspaceDir, "same.png"), Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "Compare ./same.png",
|
||||
media: [{ path: "./same.png", contentType: "image/png", workspaceDir }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
|
||||
expect(result.detectedRefs).toEqual([
|
||||
{ raw: "./same.png", type: "path", resolved: "./same.png" },
|
||||
]);
|
||||
expect(result.loadedCount).toBe(1);
|
||||
expect(result.images).toHaveLength(1);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("blocks prompt image refs outside workspace when sandbox workspaceOnly is enabled", async () => {
|
||||
// Sandbox workspaceOnly uses the bridge to validate mounted paths; ordinary
|
||||
// prompt refs outside the workspace are detected but intentionally skipped.
|
||||
@@ -675,4 +644,220 @@ describe("detectAndLoadPromptImages", () => {
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the fact hydration size limit", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-native-image-size-"));
|
||||
const imagePath = path.join(workspaceDir, "too-large.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
try {
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "describe it",
|
||||
media: [{ path: imagePath, contentType: "image/png" }],
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
maxBytes: 1,
|
||||
});
|
||||
|
||||
expect(result.loadedCount).toBe(0);
|
||||
expect(result.failedMediaCount).toBe(1);
|
||||
expect(result.skippedCount).toBe(1);
|
||||
expect(result.images).toHaveLength(0);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("hydratePromptMediaMessages", () => {
|
||||
it("hydrates queued facts in attachment order without mutating cache-stable input", async () => {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-queued-image-facts-"));
|
||||
const inboundDir = path.join(stateDir, "media", "inbound");
|
||||
const mediaId = "queued.gif";
|
||||
await fs.mkdir(inboundDir, { recursive: true });
|
||||
await fs.writeFile(path.join(inboundDir, mediaId), TINY_GIF_BUFFER);
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const message = attachRuntimePromptMediaFacts(
|
||||
{ role: "user" as const, content: [{ type: "text" as const, text: "compare" }, inlineImage] },
|
||||
[{ url: `media://inbound/${mediaId}`, contentType: "image/gif" }],
|
||||
["offloaded", "inline"],
|
||||
);
|
||||
const options = {
|
||||
workspaceDir: stateDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
};
|
||||
|
||||
try {
|
||||
const runtimeMessage = message as unknown as AgentMessage;
|
||||
const first = await hydratePromptMediaMessages([runtimeMessage], options);
|
||||
const second = await hydratePromptMediaMessages([runtimeMessage], options);
|
||||
const firstContent = (first?.[0] as unknown as { content?: unknown })?.content;
|
||||
const secondContent = (second?.[0] as unknown as { content?: unknown })?.content;
|
||||
|
||||
expect(firstContent).toEqual([
|
||||
{ type: "text", text: "compare" },
|
||||
{ type: "image", data: TINY_GIF_BUFFER.toString("base64"), mimeType: "image/gif" },
|
||||
inlineImage,
|
||||
]);
|
||||
expect(secondContent).toEqual(firstContent);
|
||||
expect(message.content).toEqual([{ type: "text", text: "compare" }, inlineImage]);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("hydrates a fact-owned user message whose content is a string", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-string-image-facts-"));
|
||||
const imagePath = path.join(workspaceDir, "photo.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const message = attachRuntimePromptMediaFacts(
|
||||
{ role: "user" as const, content: "describe it" },
|
||||
[{ path: imagePath, contentType: "image/png" }],
|
||||
) as unknown as AgentMessage;
|
||||
|
||||
try {
|
||||
const result = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
|
||||
{ type: "text", text: "describe it" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reconstructs recent facts from serialized transcript media fields", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replayed-image-"));
|
||||
const imagePath = path.join(workspaceDir, "photo.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const message = {
|
||||
role: "user" as const,
|
||||
content: "describe the replayed image",
|
||||
MediaPath: imagePath,
|
||||
MediaPaths: [imagePath],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png"],
|
||||
} as unknown as AgentMessage;
|
||||
|
||||
try {
|
||||
const result = await hydratePromptMediaMessages([message], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
|
||||
{ type: "text", text: "describe the replayed image" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves offloaded-before-inline order across serialize and restore", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-order-"));
|
||||
const offloadedPath = path.join(workspaceDir, "offloaded.png");
|
||||
const inlinePath = path.join(workspaceDir, "inline.gif");
|
||||
await fs.writeFile(offloadedPath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
await fs.writeFile(inlinePath, TINY_GIF_BUFFER);
|
||||
const inlineImage = {
|
||||
type: "image" as const,
|
||||
data: TINY_GIF_BUFFER.toString("base64"),
|
||||
mimeType: "image/gif",
|
||||
};
|
||||
const runtime = attachRuntimePromptMediaFacts(
|
||||
{ role: "user" as const, content: [{ type: "text" as const, text: "compare" }, inlineImage] },
|
||||
[{ path: offloadedPath, contentType: "image/png" }],
|
||||
["offloaded", "inline"],
|
||||
) as unknown as AgentMessage;
|
||||
const persisted = buildPersistedUserTurnMessage({
|
||||
text: "compare",
|
||||
media: [
|
||||
{ path: offloadedPath, contentType: "image/png" },
|
||||
{ path: inlinePath, contentType: "image/gif" },
|
||||
],
|
||||
mediaImageLayout: {
|
||||
slots: [
|
||||
{ kind: "offloaded", factIndex: 0 },
|
||||
{ kind: "inline", factIndex: 1 },
|
||||
],
|
||||
},
|
||||
});
|
||||
const serialized = JSON.stringify(
|
||||
mergePreparedUserTurnMessageForRuntime({
|
||||
runtimeMessage: runtime,
|
||||
preparedMessage: persisted,
|
||||
}),
|
||||
);
|
||||
const restored = JSON.parse(serialized) as AgentMessage;
|
||||
|
||||
try {
|
||||
expect(readRuntimePromptImageOrder(restored)).toBeUndefined();
|
||||
const result = await hydratePromptMediaMessages([restored], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((result[0] as unknown as { content?: unknown }).content).toEqual([
|
||||
{ type: "text", text: "compare" },
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
inlineImage,
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps duplicate fact slots across serialize and restore", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-replay-duplicates-"));
|
||||
const imagePath = path.join(workspaceDir, "same.png");
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const persisted = buildPersistedUserTurnMessage({
|
||||
text: "compare duplicates",
|
||||
media: [
|
||||
{ path: imagePath, contentType: "image/png" },
|
||||
{ path: imagePath, contentType: "image/png" },
|
||||
],
|
||||
mediaImageLayout: {
|
||||
slots: [
|
||||
{ kind: "offloaded", factIndex: 0 },
|
||||
{ kind: "inline", factIndex: 1 },
|
||||
],
|
||||
},
|
||||
});
|
||||
const serialized = JSON.stringify(
|
||||
mergePreparedUserTurnMessageForRuntime({
|
||||
runtimeMessage: {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "compare duplicates" }, inlineImage],
|
||||
} as AgentMessage,
|
||||
preparedMessage: persisted,
|
||||
}),
|
||||
);
|
||||
const restored = JSON.parse(serialized) as AgentMessage;
|
||||
|
||||
try {
|
||||
const result = await hydratePromptMediaMessages([restored], {
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
const content = (result[0] as unknown as { content?: unknown[] }).content ?? [];
|
||||
expect(content.filter((block) => (block as { type?: unknown }).type === "image")).toEqual([
|
||||
inlineImage,
|
||||
inlineImage,
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,113 @@
|
||||
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
|
||||
import { isImageMediaFact, resolveMediaFacts } from "../../../media/media-facts.js";
|
||||
import { resolveImageSanitizationLimits } from "../../image-sanitization.js";
|
||||
import { resolveAttemptWorkspaceSandbox } from "./attempt-setup.js";
|
||||
import { detectAndLoadPromptImages } from "./images.js";
|
||||
import type { RunEmbeddedAgentParams } from "./params.js";
|
||||
import {
|
||||
readPersistedMediaImageLayout,
|
||||
readPersistedPromptMediaFacts,
|
||||
} from "./prompt-image-metadata.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
function toTypeOnlyImageFact(
|
||||
fact: NonNullable<RunEmbeddedAgentParams["media"]>[number],
|
||||
hydrationSuppressed: boolean,
|
||||
): NonNullable<RunEmbeddedAgentParams["media"]>[number] {
|
||||
return {
|
||||
contentType: fact.contentType,
|
||||
kind: fact.kind === "sticker" ? "sticker" : "image",
|
||||
messageId: fact.messageId,
|
||||
transcribed: fact.transcribed,
|
||||
...(fact.hydrationSuppressed === true || hydrationSuppressed
|
||||
? { hydrationSuppressed: true }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Materializes fact-carried images before a plugin harness owns transport. */
|
||||
export async function preparePluginHarnessPromptImages(params: {
|
||||
runParams: RunEmbeddedAgentParams;
|
||||
runtime: {
|
||||
sessionId: string;
|
||||
sessionKey?: string;
|
||||
workspaceDir: string;
|
||||
model: EmbeddedRunAttemptParams["model"];
|
||||
};
|
||||
pluginHarnessOwnsTransport: boolean;
|
||||
}): Promise<{
|
||||
images: RunEmbeddedAgentParams["images"];
|
||||
imageOrder: RunEmbeddedAgentParams["imageOrder"];
|
||||
media: RunEmbeddedAgentParams["media"];
|
||||
}> {
|
||||
const { runParams, runtime } = params;
|
||||
if (!params.pluginHarnessOwnsTransport) {
|
||||
return {
|
||||
images: runParams.images,
|
||||
imageOrder: runParams.imageOrder,
|
||||
media: runParams.media,
|
||||
};
|
||||
}
|
||||
const persistedMessage =
|
||||
runParams.userTurnTranscriptRecorder?.message ??
|
||||
(await runParams.userTurnTranscriptRecorder?.resolveMessage());
|
||||
const persistedMedia = persistedMessage
|
||||
? (readPersistedPromptMediaFacts(persistedMessage) ??
|
||||
resolveMediaFacts(persistedMessage as unknown as Parameters<typeof resolveMediaFacts>[0]))
|
||||
: [];
|
||||
const hydrationMedia = persistedMedia.length > 0 ? persistedMedia : runParams.media;
|
||||
if (!hydrationMedia?.some(isImageMediaFact)) {
|
||||
return {
|
||||
images: runParams.images,
|
||||
imageOrder: runParams.imageOrder,
|
||||
media: runParams.media,
|
||||
};
|
||||
}
|
||||
|
||||
const workspace = await resolveAttemptWorkspaceSandbox({
|
||||
...runParams,
|
||||
cwd: undefined,
|
||||
sessionId: runtime.sessionId,
|
||||
sessionKey: runtime.sessionKey,
|
||||
workspaceDir: runtime.workspaceDir,
|
||||
});
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: hydrationMedia,
|
||||
mediaImageLayout: persistedMessage
|
||||
? readPersistedMediaImageLayout(persistedMessage)
|
||||
: undefined,
|
||||
workspaceDir: workspace.effectiveWorkspace,
|
||||
model: runtime.model,
|
||||
existingImages: runParams.images,
|
||||
imageOrder: runParams.imageOrder,
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
maxDimensionPx: resolveImageSanitizationLimits(runParams.config).maxDimensionPx,
|
||||
localRoots: workspace.effectiveFsWorkspaceOnly
|
||||
? [workspace.effectiveWorkspace, workspace.resolvedWorkspace]
|
||||
: undefined,
|
||||
workspaceOnly: workspace.effectiveFsWorkspaceOnly,
|
||||
sandbox:
|
||||
workspace.sandbox?.enabled && workspace.sandbox.fsBridge
|
||||
? { root: workspace.sandbox.workspaceDir, bridge: workspace.sandbox.fsBridge }
|
||||
: undefined,
|
||||
});
|
||||
if (result.failedMediaCount > 0) {
|
||||
throw new Error(
|
||||
`failed to hydrate ${result.failedMediaCount} structured image attachment(s) for plugin harness input`,
|
||||
);
|
||||
}
|
||||
const materializedFactIndexes = new Set(
|
||||
result.imageFactIndexes.filter((index): index is number => index !== null),
|
||||
);
|
||||
const retainedMedia = hydrationMedia?.map((fact, factIndex) =>
|
||||
isImageMediaFact(fact)
|
||||
? toTypeOnlyImageFact(fact, !materializedFactIndexes.has(factIndex))
|
||||
: fact,
|
||||
);
|
||||
return {
|
||||
images: result.images,
|
||||
imageOrder: result.images.length > 0 ? result.images.map(() => "inline" as const) : undefined,
|
||||
media: retainedMedia?.length ? retainedMedia : undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { normalizeMediaFacts, type MediaFact } from "../../../media/media-facts.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
|
||||
export type ImageFactIndex = number | null;
|
||||
|
||||
export type MediaImageLayout = {
|
||||
slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>;
|
||||
suppressedFactIndexes: number[];
|
||||
};
|
||||
|
||||
export function resolveLayoutInlineFactIndexes(
|
||||
layout: MediaImageLayout | undefined,
|
||||
existingImageCount: number,
|
||||
): ImageFactIndex[] | undefined {
|
||||
const factIndexes = layout?.slots.flatMap((slot) =>
|
||||
slot.kind === "inline" ? [slot.factIndex ?? null] : [],
|
||||
);
|
||||
return factIndexes?.length === existingImageCount ? factIndexes : undefined;
|
||||
}
|
||||
|
||||
export function countMissingLayoutInlineSlots(
|
||||
layout: MediaImageLayout | undefined,
|
||||
existingFactIndexes: readonly ImageFactIndex[] | undefined,
|
||||
existingImageCount: number,
|
||||
): number {
|
||||
if (!layout) {
|
||||
return 0;
|
||||
}
|
||||
const available = existingFactIndexes
|
||||
? [...existingFactIndexes]
|
||||
: Array.from({ length: existingImageCount }, () => null);
|
||||
let missing = 0;
|
||||
for (const slot of layout.slots) {
|
||||
if (slot.kind !== "inline") {
|
||||
continue;
|
||||
}
|
||||
const exactIndex =
|
||||
slot.factIndex === undefined
|
||||
? available.length > 0
|
||||
? 0
|
||||
: -1
|
||||
: available.findIndex((factIndex) => factIndex === slot.factIndex);
|
||||
const matchIndex = exactIndex >= 0 ? exactIndex : available.indexOf(null);
|
||||
if (matchIndex >= 0) {
|
||||
available.splice(matchIndex, 1);
|
||||
} else {
|
||||
missing++;
|
||||
}
|
||||
}
|
||||
return missing;
|
||||
}
|
||||
|
||||
export function readPersistedImageBlockFactIndexes(
|
||||
message: AgentMessage,
|
||||
): ImageFactIndex[] | undefined {
|
||||
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
|
||||
const value =
|
||||
meta && typeof meta === "object" && !Array.isArray(meta)
|
||||
? (meta as Record<string, unknown>).mediaImageBlockFactIndexes
|
||||
: undefined;
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined;
|
||||
}
|
||||
return value.map((entry) =>
|
||||
typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0 ? entry : null,
|
||||
);
|
||||
}
|
||||
|
||||
export function readPersistedPromptMediaFacts(message: AgentMessage): MediaFact[] | undefined {
|
||||
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
|
||||
const media =
|
||||
meta && typeof meta === "object" && !Array.isArray(meta)
|
||||
? (meta as Record<string, unknown>).media
|
||||
: undefined;
|
||||
return Array.isArray(media) ? normalizeMediaFacts(media as MediaFact[]) : undefined;
|
||||
}
|
||||
|
||||
export function readPersistedMediaImageLayout(message: AgentMessage): MediaImageLayout | undefined {
|
||||
const meta = (message as unknown as Record<string, unknown>)["__openclaw"];
|
||||
if (!meta || typeof meta !== "object" || Array.isArray(meta)) {
|
||||
return undefined;
|
||||
}
|
||||
const layout = (meta as Record<string, unknown>).mediaImageLayout;
|
||||
if (!layout || typeof layout !== "object" || Array.isArray(layout)) {
|
||||
return undefined;
|
||||
}
|
||||
const record = layout as Record<string, unknown>;
|
||||
const slots = Array.isArray(record.slots)
|
||||
? record.slots.flatMap((entry) => {
|
||||
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
||||
return [];
|
||||
}
|
||||
const slot = entry as Record<string, unknown>;
|
||||
if (slot.kind !== "inline" && slot.kind !== "offloaded") {
|
||||
return [];
|
||||
}
|
||||
const kind: MediaImageLayout["slots"][number]["kind"] = slot.kind;
|
||||
const factIndex = slot.factIndex;
|
||||
return [
|
||||
{
|
||||
kind,
|
||||
...(typeof factIndex === "number" && Number.isSafeInteger(factIndex) && factIndex >= 0
|
||||
? { factIndex }
|
||||
: {}),
|
||||
},
|
||||
];
|
||||
})
|
||||
: [];
|
||||
const suppressedFactIndexes = Array.isArray(record.suppressedFactIndexes)
|
||||
? record.suppressedFactIndexes.filter(
|
||||
(entry): entry is number =>
|
||||
typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0,
|
||||
)
|
||||
: [];
|
||||
return slots.length > 0 || suppressedFactIndexes.length > 0
|
||||
? { slots, suppressedFactIndexes }
|
||||
: undefined;
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
|
||||
import { readRuntimePromptImageFactIndexes } from "../../../media/runtime-prompt-image-provenance.js";
|
||||
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
|
||||
import { detectAndLoadPromptImages } from "./images.js";
|
||||
import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js";
|
||||
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
|
||||
|
||||
describe("plugin harness prompt media", () => {
|
||||
it("hydrates plugin images and preserves serialized replay order with non-image facts", async () => {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-harness-media-"));
|
||||
const workspaceDir = path.join(stateDir, "workspace");
|
||||
const inboundDir = path.join(stateDir, "media", "inbound");
|
||||
const mediaId = "photo.png";
|
||||
const imagePath = path.join(inboundDir, mediaId);
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
await fs.mkdir(inboundDir, { recursive: true });
|
||||
await fs.writeFile(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
const documentFact = {
|
||||
path: path.join(workspaceDir, "misleading.png"),
|
||||
contentType: "application/pdf",
|
||||
kind: "document" as const,
|
||||
};
|
||||
const input = {
|
||||
runParams: {
|
||||
agentId: "main",
|
||||
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
|
||||
imageOrder: ["offloaded"],
|
||||
media: [documentFact, { url: `media://inbound/${mediaId}`, contentType: "image/png" }],
|
||||
sessionId: "session-1",
|
||||
userTurnTranscriptRecorder: {
|
||||
message: {
|
||||
role: "user",
|
||||
content: "inspect",
|
||||
MediaPaths: [imagePath, documentFact.path],
|
||||
MediaTypes: ["image/png", "application/pdf"],
|
||||
__openclaw: {
|
||||
mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
runtime: {
|
||||
model: { input: ["text", "image"] },
|
||||
sessionId: "session-1",
|
||||
workspaceDir,
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0];
|
||||
|
||||
try {
|
||||
const result = await preparePluginHarnessPromptImages(input);
|
||||
|
||||
expect(result.images).toEqual([
|
||||
{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" },
|
||||
]);
|
||||
expect(readRuntimePromptImageFactIndexes(result.images ?? [])).toEqual([0]);
|
||||
expect(result.imageOrder).toEqual(["inline"]);
|
||||
expect(result.media?.[0]).toMatchObject({ contentType: "image/png", kind: "image" });
|
||||
expect(result.media?.[0]).not.toHaveProperty("path");
|
||||
expect(result.media?.[0]).not.toHaveProperty("url");
|
||||
expect(result.media?.[1]).toMatchObject(documentFact);
|
||||
|
||||
const serialized = JSON.stringify(result);
|
||||
const restored = JSON.parse(serialized) as typeof result;
|
||||
const replay = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: restored.media,
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: restored.images,
|
||||
imageOrder: restored.imageOrder,
|
||||
});
|
||||
expect(replay.failedMediaCount).toBe(0);
|
||||
expect(replay.images).toEqual(result.images);
|
||||
expect(replay.imageFactIndexes).toEqual([0]);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces a failed image hydration before plugin dispatch", async () => {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-harness-failed-media-"));
|
||||
try {
|
||||
await expect(
|
||||
preparePluginHarnessPromptImages({
|
||||
runParams: {
|
||||
agentId: "main",
|
||||
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
|
||||
imageOrder: ["offloaded"],
|
||||
media: [{ path: path.join(workspaceDir, "missing.png"), contentType: "image/png" }],
|
||||
sessionId: "session-failed",
|
||||
},
|
||||
runtime: {
|
||||
model: { input: ["text", "image"] },
|
||||
sessionId: "session-failed",
|
||||
workspaceDir,
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
|
||||
).rejects.toThrow("failed to hydrate 1 structured image attachment");
|
||||
} finally {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces an unsuppressed identity-less inline fact with no image block", async () => {
|
||||
await expect(
|
||||
preparePluginHarnessPromptImages({
|
||||
runParams: {
|
||||
agentId: "main",
|
||||
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
|
||||
imageOrder: ["inline"],
|
||||
media: [{ kind: "image" }],
|
||||
sessionId: "session-missing-inline",
|
||||
},
|
||||
runtime: {
|
||||
model: { input: ["text", "image"] },
|
||||
sessionId: "session-missing-inline",
|
||||
workspaceDir: "/tmp",
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
|
||||
).rejects.toThrow("failed to hydrate 1 structured image attachment");
|
||||
});
|
||||
|
||||
it("surfaces a fact-owned image dropped during host sanitization", async () => {
|
||||
await expect(
|
||||
preparePluginHarnessPromptImages({
|
||||
runParams: {
|
||||
agentId: "main",
|
||||
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
|
||||
images: [{ type: "image", data: "%%%", mimeType: "image/png" }],
|
||||
imageOrder: ["inline"],
|
||||
media: [{ kind: "image" }],
|
||||
sessionId: "session-sanitize-failed",
|
||||
},
|
||||
runtime: {
|
||||
model: { input: ["text", "image"] },
|
||||
sessionId: "session-sanitize-failed",
|
||||
workspaceDir: "/tmp",
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
|
||||
).rejects.toThrow("failed to hydrate 1 structured image attachment");
|
||||
});
|
||||
|
||||
it("surfaces inline sanitization failure when a preceding plugin image fact is suppressed", async () => {
|
||||
await expect(
|
||||
preparePluginHarnessPromptImages({
|
||||
runParams: {
|
||||
agentId: "main",
|
||||
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
|
||||
images: [{ type: "image", data: "%%%", mimeType: "image/png" }],
|
||||
imageOrder: ["inline"],
|
||||
media: [
|
||||
{
|
||||
path: "/tmp/described-missing.png",
|
||||
contentType: "image/png",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
{ path: "/tmp/inline.png", contentType: "image/png" },
|
||||
],
|
||||
sessionId: "session-suppressed-before-inline",
|
||||
},
|
||||
runtime: {
|
||||
model: { input: ["text", "image"] },
|
||||
sessionId: "session-suppressed-before-inline",
|
||||
workspaceDir: "/tmp",
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
|
||||
).rejects.toThrow("failed to hydrate 1 structured image attachment");
|
||||
});
|
||||
|
||||
it("retains an intentionally non-hydrating remote-only image as a type-only fact", async () => {
|
||||
const media = buildInboundMediaNoteProjection({
|
||||
MediaPaths: [""],
|
||||
MediaUrls: ["https://example.com/described.png"],
|
||||
MediaTypes: ["image/png"],
|
||||
MediaUnderstanding: [
|
||||
{
|
||||
kind: "image.description",
|
||||
attachmentIndex: 0,
|
||||
text: "already described",
|
||||
provider: "test",
|
||||
},
|
||||
],
|
||||
}).media;
|
||||
const result = await preparePluginHarnessPromptImages({
|
||||
runParams: {
|
||||
agentId: "main",
|
||||
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
|
||||
media,
|
||||
sessionId: "session-described",
|
||||
},
|
||||
runtime: {
|
||||
model: { input: ["text", "image"] },
|
||||
sessionId: "session-described",
|
||||
workspaceDir: "/tmp",
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
|
||||
|
||||
expect(result.images).toEqual([]);
|
||||
expect(result.media?.[0]).toMatchObject({
|
||||
contentType: "image/png",
|
||||
kind: "image",
|
||||
hydrationSuppressed: true,
|
||||
});
|
||||
expect(result.media?.[0]).not.toHaveProperty("path");
|
||||
expect(result.media?.[0]).not.toHaveProperty("url");
|
||||
});
|
||||
|
||||
it("retains layout-derived suppression after plugin host materialization", async () => {
|
||||
const inlineImage = { type: "image" as const, data: TINY_PNG_BASE64, mimeType: "image/png" };
|
||||
const result = await preparePluginHarnessPromptImages({
|
||||
runParams: {
|
||||
agentId: "main",
|
||||
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
|
||||
images: [inlineImage],
|
||||
imageOrder: ["inline"],
|
||||
media: [
|
||||
{ path: "/tmp/described.png", contentType: "image/png" },
|
||||
{ path: "/tmp/inline.png", contentType: "image/png" },
|
||||
],
|
||||
sessionId: "session-layout-suppressed",
|
||||
userTurnTranscriptRecorder: {
|
||||
message: {
|
||||
role: "user",
|
||||
content: "compare",
|
||||
MediaPaths: ["/tmp/described.png", "/tmp/inline.png"],
|
||||
MediaTypes: ["image/png", "image/png"],
|
||||
__openclaw: {
|
||||
mediaImageLayout: {
|
||||
slots: [{ kind: "inline", factIndex: 1 }],
|
||||
suppressedFactIndexes: [0],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
runtime: {
|
||||
model: { input: ["text", "image"] },
|
||||
sessionId: "session-layout-suppressed",
|
||||
workspaceDir: "/tmp",
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
|
||||
|
||||
expect(result.images).toEqual([inlineImage]);
|
||||
expect(result.imageOrder).toEqual(["inline"]);
|
||||
expect(result.media?.[0]).toMatchObject({ kind: "image", hydrationSuppressed: true });
|
||||
expect(result.media?.[1]).toMatchObject({ kind: "image" });
|
||||
expect(result.media?.[1]).not.toHaveProperty("hydrationSuppressed");
|
||||
});
|
||||
|
||||
it("keeps unsupported native images as aligned type-only facts", async () => {
|
||||
const media = [
|
||||
{ path: "/tmp/photo.png", contentType: "image/png" },
|
||||
{ path: "/tmp/inferred.png", kind: "unknown" as const },
|
||||
];
|
||||
const result = await preparePluginHarnessPromptImages({
|
||||
runParams: {
|
||||
agentId: "main",
|
||||
config: { agents: { defaults: { sandbox: { mode: "off" } } } },
|
||||
media,
|
||||
sessionId: "session-text-only",
|
||||
},
|
||||
runtime: {
|
||||
model: { input: ["text"] },
|
||||
sessionId: "session-text-only",
|
||||
workspaceDir: "/tmp",
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
|
||||
|
||||
expect(result.images).toEqual([]);
|
||||
expect(result.media?.[0]).toMatchObject({ contentType: "image/png" });
|
||||
expect(result.media?.[0]).not.toHaveProperty("path");
|
||||
expect(result.media?.[1]).toMatchObject({ kind: "image" });
|
||||
expect(result.media?.[1]).not.toHaveProperty("path");
|
||||
});
|
||||
|
||||
it("leaves facts untouched when the native harness owns transport", async () => {
|
||||
const media = [{ path: "/tmp/photo.png", contentType: "image/png" }];
|
||||
const result = await preparePluginHarnessPromptImages({
|
||||
runParams: { media },
|
||||
runtime: {},
|
||||
pluginHarnessOwnsTransport: false,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
|
||||
|
||||
expect(result).toEqual({ images: undefined, imageOrder: undefined, media });
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS,
|
||||
} from "./lane-runtime.js";
|
||||
import type { RunEmbeddedAgentParams } from "./params.js";
|
||||
import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js";
|
||||
import { resolveSkillWorkshopAttemptParams } from "./skill-workshop-attempt-params.js";
|
||||
import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptTrajectoryRecorder } from "./types.js";
|
||||
|
||||
@@ -161,6 +162,11 @@ export async function dispatchEmbeddedRunAttempt(input: {
|
||||
};
|
||||
|
||||
let cancellationRequested = false;
|
||||
const promptMedia = await preparePluginHarnessPromptImages({
|
||||
runParams: params,
|
||||
runtime,
|
||||
pluginHarnessOwnsTransport: control.pluginHarnessOwnsTransport,
|
||||
});
|
||||
const attemptParams: EmbeddedRunAttemptParams = {
|
||||
operation: "attempt",
|
||||
sessionId: runtime.sessionId,
|
||||
@@ -223,8 +229,9 @@ export async function dispatchEmbeddedRunAttempt(input: {
|
||||
skipPreparedUserTurnMessage: runtime.skipPreparedUserTurnMessage,
|
||||
currentInboundEventKind: params.currentInboundEventKind,
|
||||
currentInboundContext: params.currentInboundContext,
|
||||
images: params.images,
|
||||
imageOrder: params.imageOrder,
|
||||
images: promptMedia.images,
|
||||
imageOrder: promptMedia.imageOrder,
|
||||
media: promptMedia.media,
|
||||
clientTools: params.clientTools,
|
||||
disableTools: params.disableTools,
|
||||
provider: runtime.provider,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import type { ImageContent, TextContent } from "../../llm/types.js";
|
||||
import { attachRuntimePromptMediaFacts, type MediaFact } from "../../media/media-facts.js";
|
||||
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
|
||||
import { readRuntimePromptImageFactIndexes } from "../../media/runtime-prompt-image-provenance.js";
|
||||
import { attachRuntimeUserTurnTranscriptContext } from "../../sessions/user-turn-transcript-runtime-context.js";
|
||||
import type {
|
||||
PersistedUserTurnMessage,
|
||||
@@ -89,6 +91,21 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
|
||||
return [{ type: "text", text }, ...(images ?? [])];
|
||||
}
|
||||
|
||||
private createUserMessage(text: string, images?: ImageContent[]): PersistedUserTurnMessage {
|
||||
const message = {
|
||||
role: "user",
|
||||
content: this.createUserContent(text, images),
|
||||
timestamp: Date.now(),
|
||||
} satisfies PersistedUserTurnMessage;
|
||||
const imageFactIndexes = readRuntimePromptImageFactIndexes(images);
|
||||
return imageFactIndexes
|
||||
? ({
|
||||
...message,
|
||||
__openclaw: { mediaImageBlockFactIndexes: imageFactIndexes },
|
||||
} as unknown as PersistedUserTurnMessage)
|
||||
: message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a prompt to the agent.
|
||||
* - Handles extension commands immediately, even during streaming
|
||||
@@ -188,11 +205,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
|
||||
messages = [];
|
||||
|
||||
// Add user message
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: this.createUserContent(expandedText, currentImages),
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
messages.push(this.createUserMessage(expandedText, currentImages));
|
||||
|
||||
// Inject any pending "nextTurn" messages as context alongside the user message
|
||||
for (const msg of this.pendingNextTurnMessages) {
|
||||
@@ -322,6 +335,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
|
||||
images?: ImageContent[],
|
||||
userTurnTranscriptRecorder?: UserTurnTranscriptRecorder,
|
||||
media?: MediaFact[],
|
||||
imageOrder?: PromptImageOrderEntry[],
|
||||
): Promise<void> {
|
||||
// Check for extension commands (cannot be queued)
|
||||
if (text.startsWith("/")) {
|
||||
@@ -340,6 +354,7 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
|
||||
? { message: preparedMessage, recorder: userTurnTranscriptRecorder }
|
||||
: undefined,
|
||||
media,
|
||||
imageOrder,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -374,16 +389,13 @@ export abstract class AgentSessionPrompting extends AgentSessionBase {
|
||||
recorder: UserTurnTranscriptRecorder;
|
||||
},
|
||||
media?: MediaFact[],
|
||||
imageOrder?: PromptImageOrderEntry[],
|
||||
): Promise<void> {
|
||||
this.steeringMessages.push(text);
|
||||
this.emitQueueUpdate();
|
||||
const runtimeMessage = {
|
||||
role: "user",
|
||||
content: this.createUserContent(text, images),
|
||||
timestamp: Date.now(),
|
||||
} satisfies PersistedUserTurnMessage;
|
||||
const runtimeMessage = this.createUserMessage(text, images);
|
||||
const promptMessage = media?.length
|
||||
? attachRuntimePromptMediaFacts(runtimeMessage, media)
|
||||
? attachRuntimePromptMediaFacts(runtimeMessage, media, imageOrder)
|
||||
: runtimeMessage;
|
||||
this.agent.steer(
|
||||
transcriptContext
|
||||
|
||||
@@ -4,7 +4,9 @@ import { createAssistantMessageEventStream, type AssistantMessage } from "opencl
|
||||
import { Type } from "typebox";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getStreamLlmRuntime } from "../../llm/model-runtime-binding.js";
|
||||
import type { Model, SimpleStreamOptions } from "../../llm/types.js";
|
||||
import type { ImageContent, Model, SimpleStreamOptions } from "../../llm/types.js";
|
||||
import { readRuntimePromptImageOrder } from "../../media/media-facts.js";
|
||||
import { finalizeRuntimePromptImages } from "../../media/runtime-prompt-image-provenance.js";
|
||||
import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js";
|
||||
import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js";
|
||||
|
||||
@@ -295,8 +297,13 @@ describe("AgentSession queued user turns", () => {
|
||||
const session = await createSessionFromManager(SessionManager.inMemory());
|
||||
const steer = vi.spyOn(session.agent, "steer").mockImplementation(() => undefined);
|
||||
const media = [{ path: "/tmp/a.png", contentType: "image/png" }];
|
||||
const imageOrder = ["inline"] as const;
|
||||
const image: ImageContent = { type: "image", data: "aW1hZ2U=", mimeType: "image/png" };
|
||||
const { images } = finalizeRuntimePromptImages([{ image, factIndex: 0 }]);
|
||||
|
||||
await session.steer("[media attached: /tmp/a.png (image/png)]", undefined, undefined, media);
|
||||
await session.steer("[media attached: /tmp/a.png (image/png)]", images, undefined, media, [
|
||||
...imageOrder,
|
||||
]);
|
||||
|
||||
const runtimeMessage = steer.mock.calls[0]?.[0];
|
||||
expect(runtimeMessage).toBeDefined();
|
||||
@@ -304,12 +311,16 @@ describe("AgentSession queued user turns", () => {
|
||||
(symbol) => Symbol.keyFor(symbol) === "openclaw.runtimePromptMediaFacts",
|
||||
);
|
||||
expect(mediaSymbol).toBeDefined();
|
||||
if (!mediaSymbol) {
|
||||
throw new Error("expected runtime prompt media symbol");
|
||||
if (!runtimeMessage || !mediaSymbol) {
|
||||
throw new Error("expected runtime prompt media message and symbol");
|
||||
}
|
||||
expect((runtimeMessage as unknown as Record<PropertyKey, unknown>)[mediaSymbol]).toEqual([
|
||||
expect.objectContaining({ path: "/tmp/a.png", contentType: "image/png", kind: "image" }),
|
||||
]);
|
||||
expect(readRuntimePromptImageOrder(runtimeMessage)).toEqual(imageOrder);
|
||||
expect((runtimeMessage as unknown as Record<string, unknown>)["__openclaw"]).toEqual({
|
||||
mediaImageBlockFactIndexes: [0],
|
||||
});
|
||||
expect(JSON.stringify(runtimeMessage)).not.toContain("runtimePromptMediaFacts");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -111,7 +111,7 @@ describe("buildInboundMediaNote", () => {
|
||||
});
|
||||
|
||||
it("keeps image attachments after image descriptions are added", () => {
|
||||
const note = buildInboundMediaNote({
|
||||
const projection = buildInboundMediaNoteProjection({
|
||||
MediaPaths: ["/tmp/photo.png"],
|
||||
MediaUrls: ["https://example.com/photo.png"],
|
||||
MediaTypes: ["image/png"],
|
||||
@@ -124,9 +124,20 @@ describe("buildInboundMediaNote", () => {
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(note).toBe(
|
||||
expect(projection.text).toBe(
|
||||
"[media attached: /tmp/photo.png (image/png) | https://example.com/photo.png]",
|
||||
);
|
||||
expect(projection.media).toEqual([
|
||||
{
|
||||
path: "/tmp/photo.png",
|
||||
url: "https://example.com/photo.png",
|
||||
contentType: "image/png",
|
||||
kind: "image",
|
||||
transcribed: false,
|
||||
messageId: undefined,
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps image attachments when image understanding succeeds via decisions", () => {
|
||||
|
||||
@@ -131,12 +131,20 @@ function collectTranscribedAudioAttachmentIndices(
|
||||
return transcribedAudioIndices;
|
||||
}
|
||||
|
||||
function collectDescribedImageAttachmentIndices(ctx: MsgContext): Set<number> {
|
||||
return new Set(
|
||||
ctx.MediaUnderstanding?.flatMap((output) =>
|
||||
output.kind === "image.description" ? [output.attachmentIndex] : [],
|
||||
) ?? [],
|
||||
);
|
||||
}
|
||||
|
||||
type InboundMediaNoteProjection = {
|
||||
text?: string;
|
||||
media: MediaFact[];
|
||||
};
|
||||
|
||||
/** Formats prompt-visible attachment text and retains the facts represented by it. */
|
||||
/** Formats prompt-visible attachment text and retains facts that still need native hydration. */
|
||||
export function buildInboundMediaNoteProjection(ctx: MsgContext): InboundMediaNoteProjection {
|
||||
// Attachment indices follow MediaPaths/MediaUrls ordering as supplied by the channel.
|
||||
const pathsFromArray = Array.isArray(ctx.MediaPaths) ? ctx.MediaPaths : undefined;
|
||||
@@ -196,14 +204,22 @@ export function buildInboundMediaNoteProjection(ctx: MsgContext): InboundMediaNo
|
||||
return { media: [] };
|
||||
}
|
||||
const facts = resolveMediaFacts(ctx);
|
||||
const describedImageIndices = collectDescribedImageAttachmentIndices(ctx);
|
||||
const media = normalizeMediaFacts(
|
||||
entries.map((entry) => ({
|
||||
...facts[entry.index],
|
||||
path: entry.path,
|
||||
url: entry.url,
|
||||
contentType: entry.type,
|
||||
})),
|
||||
entries.map((entry) =>
|
||||
Object.assign({}, facts[entry.index], {
|
||||
path: entry.path,
|
||||
url: entry.url,
|
||||
contentType: entry.type,
|
||||
}),
|
||||
),
|
||||
);
|
||||
for (const [position, entry] of entries.entries()) {
|
||||
const fact = media[position];
|
||||
if (fact && describedImageIndices.has(entry.index)) {
|
||||
fact.hydrationSuppressed = true;
|
||||
}
|
||||
}
|
||||
if (entries.length === 1) {
|
||||
return {
|
||||
text: formatMediaAttachedLine({
|
||||
|
||||
@@ -1419,6 +1419,7 @@ export async function runReplyAgent(params: {
|
||||
steeringMode: "all",
|
||||
isInboundUserMessage: true,
|
||||
...(followupRun.images?.length ? { images: followupRun.images } : {}),
|
||||
...(followupRun.imageOrder?.length ? { imageOrder: followupRun.imageOrder } : {}),
|
||||
...(followupRun.media?.length ? { media: followupRun.media } : {}),
|
||||
...(turnAdoptionLifecycle ? { waitForTranscriptCommit: true } : {}),
|
||||
...(resolvedQueue.debounceMs !== undefined ? { debounceMs: resolvedQueue.debounceMs } : {}),
|
||||
|
||||
@@ -137,6 +137,7 @@ function appendOrderedImages(params: {
|
||||
function resolveMergedTurnImages(entries: OrderedTurnImage[]): {
|
||||
images?: ImageContent[];
|
||||
imageOrder?: PromptImageOrderEntry[];
|
||||
imageSourceIndexes?: Array<number | undefined>;
|
||||
} {
|
||||
if (entries.length === 0) {
|
||||
return {};
|
||||
@@ -151,10 +152,14 @@ function resolveMergedTurnImages(entries: OrderedTurnImage[]): {
|
||||
return left.sequence - right.sequence;
|
||||
});
|
||||
const images = merged.flatMap((entry) => (entry.image ? [entry.image] : []));
|
||||
return {
|
||||
const result = {
|
||||
...(images.length > 0 ? { images } : {}),
|
||||
imageOrder: merged.map((entry) => entry.imageOrder),
|
||||
};
|
||||
Object.defineProperty(result, "imageSourceIndexes", {
|
||||
value: merged.map((entry) => entry.sourceIndex),
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Resolves current-turn image attachments that were not already described by media understanding. */
|
||||
@@ -167,6 +172,7 @@ export async function resolveCurrentTurnImages(params: {
|
||||
}): Promise<{
|
||||
images?: ImageContent[];
|
||||
imageOrder?: PromptImageOrderEntry[];
|
||||
imageSourceIndexes?: Array<number | undefined>;
|
||||
}> {
|
||||
const entries: OrderedTurnImage[] = [];
|
||||
appendOrderedImages({
|
||||
|
||||
@@ -1399,6 +1399,13 @@ describe("runPreparedReply media-only handling", () => {
|
||||
expect(call.followupRun.images).toBeUndefined();
|
||||
expect(call.followupRun.imageOrder).toBeUndefined();
|
||||
expect(call.followupRun.prompt).toContain("a tiny dot image");
|
||||
expect(
|
||||
(
|
||||
call.followupRun.userTurnTranscriptRecorder?.message as unknown as Record<string, unknown>
|
||||
)?.["__openclaw"],
|
||||
).toMatchObject({
|
||||
mediaImageLayout: { slots: [], suppressedFactIndexes: [0, 1] },
|
||||
});
|
||||
});
|
||||
|
||||
it("rehydrates only current MediaPaths missing image understanding", async () => {
|
||||
@@ -1462,6 +1469,16 @@ describe("runPreparedReply media-only handling", () => {
|
||||
mimeType: "image/png",
|
||||
},
|
||||
]);
|
||||
expect(
|
||||
(
|
||||
call.followupRun.userTurnTranscriptRecorder?.message as unknown as Record<string, unknown>
|
||||
)?.["__openclaw"],
|
||||
).toMatchObject({
|
||||
mediaImageLayout: {
|
||||
slots: [{ kind: "inline", factIndex: 1 }],
|
||||
suppressedFactIndexes: [0],
|
||||
},
|
||||
});
|
||||
expect(call.followupRun.imageOrder).toEqual(["inline"]);
|
||||
expect(call.followupRun.prompt).toContain("a tiny dot image");
|
||||
});
|
||||
|
||||
@@ -43,7 +43,7 @@ import { measureDiagnosticsTimelineSpan } from "../../infra/diagnostics-timeline
|
||||
import { isFastTestRuntimeEnv } from "../../infra/env.js";
|
||||
import { resolveHeartbeatRunScope } from "../../infra/heartbeat-run-scope.js";
|
||||
import type { ExtractedFileImage } from "../../media-understanding/extracted-file-images.js";
|
||||
import { resolveMediaFacts, type MediaFact } from "../../media/media-facts.js";
|
||||
import { isImageMediaFact, resolveMediaFacts, type MediaFact } from "../../media/media-facts.js";
|
||||
import { clearCommandLane, getQueueSize } from "../../process/command-queue.js";
|
||||
import {
|
||||
isAcpSessionKey,
|
||||
@@ -54,6 +54,7 @@ import { MEDIA_ONLY_USER_TEXT } from "../../sessions/user-turn-media.js";
|
||||
import {
|
||||
createUserTurnTranscriptRecorder,
|
||||
resolvePersistedUserTurnText,
|
||||
type UserTurnInput,
|
||||
} from "../../sessions/user-turn-transcript.js";
|
||||
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
|
||||
import type { SilentReplyConversationType } from "../../shared/silent-reply-policy.js";
|
||||
@@ -162,6 +163,73 @@ type InternalGetReplyOptions = BaseInternalGetReplyOptions & {
|
||||
type AgentDefaults = NonNullable<OpenClawConfig["agents"]>["defaults"];
|
||||
type ExecOverrides = Pick<ExecToolDefaults, "host" | "security" | "ask" | "node" | "nodeCwd">;
|
||||
const EPOCH_MILLISECONDS_THRESHOLD = 1_000_000_000_000;
|
||||
function buildPersistedMediaImageLayout(params: {
|
||||
ctx: MsgContext;
|
||||
media: readonly MediaFact[];
|
||||
ctxMediaCount: number;
|
||||
imageOrder?: readonly ("inline" | "offloaded")[];
|
||||
imageSourceIndexes?: readonly (number | undefined)[];
|
||||
}): NonNullable<UserTurnInput["mediaImageLayout"]> | undefined {
|
||||
const describedAttachmentIndexes = new Set(
|
||||
params.ctx.MediaUnderstanding?.flatMap((output) =>
|
||||
output.kind === "image.description" ? [output.attachmentIndex] : [],
|
||||
) ?? [],
|
||||
);
|
||||
const suppressedFactIndexes: number[] = [];
|
||||
const imageFactIndexes: number[] = [];
|
||||
for (const [factIndex, fact] of params.media.entries()) {
|
||||
if (!isImageMediaFact(fact)) {
|
||||
continue;
|
||||
}
|
||||
imageFactIndexes.push(factIndex);
|
||||
if (
|
||||
(factIndex < params.ctxMediaCount && describedAttachmentIndexes.has(factIndex)) ||
|
||||
fact.hydrationSuppressed === true
|
||||
) {
|
||||
suppressedFactIndexes.push(factIndex);
|
||||
}
|
||||
}
|
||||
if (imageFactIndexes.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const suppressed = new Set(suppressedFactIndexes);
|
||||
const used = new Set<number>();
|
||||
const unsuppressedFactCount = imageFactIndexes.filter((index) => !suppressed.has(index)).length;
|
||||
const canInferByPosition = unsuppressedFactCount === (params.imageOrder?.length ?? 0);
|
||||
const takeNextFactIndex = (): number | undefined =>
|
||||
imageFactIndexes.find((index) => !suppressed.has(index) && !used.has(index));
|
||||
const slots = (params.imageOrder ?? []).map((kind, index) => {
|
||||
const sourceIndex = params.imageSourceIndexes?.[index];
|
||||
const sourceFact = sourceIndex === undefined ? undefined : params.media[sourceIndex];
|
||||
const factIndex =
|
||||
sourceIndex !== undefined
|
||||
? sourceFact &&
|
||||
isImageMediaFact(sourceFact) &&
|
||||
!suppressed.has(sourceIndex) &&
|
||||
!used.has(sourceIndex)
|
||||
? sourceIndex
|
||||
: undefined
|
||||
: canInferByPosition
|
||||
? takeNextFactIndex()
|
||||
: undefined;
|
||||
if (factIndex !== undefined) {
|
||||
used.add(factIndex);
|
||||
}
|
||||
return factIndex === undefined ? { kind } : { kind, factIndex };
|
||||
});
|
||||
for (const factIndex of imageFactIndexes) {
|
||||
if (!suppressed.has(factIndex) && !used.has(factIndex)) {
|
||||
slots.push({ kind: "offloaded", factIndex });
|
||||
}
|
||||
}
|
||||
if (slots.length === 0 && suppressedFactIndexes.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
slots,
|
||||
...(suppressedFactIndexes.length > 0 ? { suppressedFactIndexes } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function hasResolvedThinkingCatalogEntry(params: {
|
||||
catalog?: readonly ThinkingCatalogEntry[];
|
||||
@@ -1466,7 +1534,15 @@ export async function runPreparedReply(
|
||||
: undefined);
|
||||
setChannelSourceTurnId(sessionCtx, sourceTurnId);
|
||||
const persistGroupSender = replyRoute.chatType === "group" || replyRoute.chatType === "channel";
|
||||
const userTurnMediaForPersistence = [...resolveMediaFacts(ctx), ...(opts?.media ?? [])];
|
||||
const ctxMediaForPersistence = resolveMediaFacts(ctx);
|
||||
const userTurnMediaForPersistence = [...ctxMediaForPersistence, ...(opts?.media ?? [])];
|
||||
const mediaImageLayout = buildPersistedMediaImageLayout({
|
||||
ctx,
|
||||
media: userTurnMediaForPersistence,
|
||||
ctxMediaCount: ctxMediaForPersistence.length,
|
||||
imageOrder: currentTurnImages.imageOrder,
|
||||
imageSourceIndexes: currentTurnImages.imageSourceIndexes,
|
||||
});
|
||||
const inputProvenance = ctx.InputProvenance ?? sessionCtx.InputProvenance;
|
||||
const userTurnTimestamp = normalizeMessageTimestampMs(ctx.Timestamp);
|
||||
// prompt-prelude substitutes MEDIA_ONLY_USER_TEXT as transcriptBody for
|
||||
@@ -1518,6 +1594,7 @@ export async function runPreparedReply(
|
||||
: {}),
|
||||
...(transport ? { transport } : {}),
|
||||
...(userTurnMediaForPersistence.length > 0 ? { media: userTurnMediaForPersistence } : {}),
|
||||
...(mediaImageLayout ? { mediaImageLayout } : {}),
|
||||
// Persist the message's own arrival timestamp so the single
|
||||
// LLM-boundary stamping site (normalizeMessagesForLlmBoundary) can
|
||||
// derive a stable per-message `[DOW YYYY-MM-DD HH:MM TZ]` prefix that
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
} from "../../logging/diagnostic-run-activity.js";
|
||||
import { diagnosticLogger as diag } from "../../logging/diagnostic-runtime.js";
|
||||
import type { MediaFact } from "../../media/media-facts.js";
|
||||
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
|
||||
import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.types.js";
|
||||
import { resolveGlobalSingleton } from "../../shared/global-singleton.js";
|
||||
import { resolveTimerTimeoutMs } from "../../shared/number-coercion.js";
|
||||
@@ -41,6 +42,7 @@ export type ReplyBackendQueueMessageOptions = {
|
||||
debounceMs?: number;
|
||||
/** Ordered current-turn images to inject with the steering text. */
|
||||
images?: ImageContent[];
|
||||
imageOrder?: PromptImageOrderEntry[];
|
||||
/** Ordered facts represented by attachment text in this steering prompt. */
|
||||
media?: MediaFact[];
|
||||
deliveryTimeoutMs?: number;
|
||||
|
||||
@@ -232,10 +232,8 @@ describe("parseMessageWithAttachments", () => {
|
||||
expect(ref.mimeType).toBe("application/pdf");
|
||||
expect(ref.label).toBe("report.pdf");
|
||||
expect(ref.mediaRef).toMatch(/^media:\/\/inbound\//);
|
||||
// Non-image offloads MUST NOT inject a media://URI into the message —
|
||||
// the caller is responsible for routing offloadedRefs[].path into
|
||||
// ctx.MediaPaths so the workspace stage surfaces a real path.
|
||||
expect(parsed.message).toBe("read this");
|
||||
expect(parsed.message).toBe(`read this\n[media attached: ${ref.mediaRef}]`);
|
||||
expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message);
|
||||
expect(saveMediaBufferMock).toHaveBeenCalledOnce();
|
||||
expect(savedMime()).toBe("application/pdf");
|
||||
expect(logs).toHaveLength(0);
|
||||
@@ -251,7 +249,10 @@ describe("parseMessageWithAttachments", () => {
|
||||
expect(parsed.offloadedRefs).toHaveLength(1);
|
||||
expect(parsed.offloadedRefs[0]?.mimeType).toBe("application/octet-stream");
|
||||
expect(savedMime()).toBe("application/octet-stream");
|
||||
expect(parsed.message).toBe("take a look");
|
||||
expect(parsed.message).toBe(
|
||||
`take a look\n[media attached: ${parsed.offloadedRefs[0]?.mediaRef}]`,
|
||||
);
|
||||
expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message);
|
||||
expect(logs).toHaveLength(0);
|
||||
});
|
||||
|
||||
@@ -272,15 +273,12 @@ describe("parseMessageWithAttachments", () => {
|
||||
expect(parsed.imageOrder).toEqual(["inline"]);
|
||||
});
|
||||
|
||||
it("excludes non-image offloads from imageOrder in mixed batches", async () => {
|
||||
it("keeps mixed image/PDF markers in normal and image-stripped routing order", async () => {
|
||||
// Regression: a prior revision pushed "offloaded" for every offload,
|
||||
// including non-image files. In a [non-image, inline, offloaded-image]
|
||||
// batch that produced imageOrder=["offloaded","inline","offloaded"] even
|
||||
// though only one `[media attached: media://...]` line is ever appended
|
||||
// to the prompt (for the image offload). extractTrailingAttachmentMediaUris
|
||||
// then read count=2 against one trailing URI, and
|
||||
// mergePromptAttachmentImages placed the single offloaded image into the
|
||||
// first "offloaded" slot — swapping it ahead of the inline image.
|
||||
// though only one image offload existed. Structural facts and imageOrder
|
||||
// must agree so hydration cannot swap it ahead of the inline image.
|
||||
const pdf = Buffer.from("%PDF-1.4\n").toString("base64");
|
||||
const bigPng = Buffer.alloc(2_100_000);
|
||||
bigPng.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0);
|
||||
@@ -295,12 +293,16 @@ describe("parseMessageWithAttachments", () => {
|
||||
"image/png",
|
||||
]);
|
||||
expect(parsed.imageOrder).toEqual(["inline", "offloaded"]);
|
||||
// The offloaded-image URI is the sole trailing media:// line, matching
|
||||
// imageOrder's single "offloaded" slot.
|
||||
const pdfRef = expectDefined(parsed.offloadedRefs[0], "offloaded PDF ref");
|
||||
const imageRef = expectDefined(parsed.offloadedRefs[1], "offloaded image ref");
|
||||
expect(parsed.message).toBe(
|
||||
`x\n[media attached: ${pdfRef.mediaRef}]\n[media attached: ${imageRef.mediaRef}]`,
|
||||
);
|
||||
expect(parsed.messageWithoutOffloadedImageRefs).toBe(`x\n[media attached: ${pdfRef.mediaRef}]`);
|
||||
const trailingMediaLines = parsed.message
|
||||
.split("\n")
|
||||
.filter((line) => line.trim().startsWith("[media attached: media://inbound/"));
|
||||
expect(trailingMediaLines).toHaveLength(1);
|
||||
expect(trailingMediaLines).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("rejects oversized images before offload", async () => {
|
||||
@@ -443,6 +445,7 @@ describe("parseMessageWithAttachments validation errors", () => {
|
||||
it("passes through unchanged on text-only session with no attachments", async () => {
|
||||
const { parsed } = await parseWithWarnings("hello", [], { supportsInlineImages: false });
|
||||
expect(parsed.message).toBe("hello");
|
||||
expect(parsed.messageWithoutOffloadedImageRefs).toBe("hello");
|
||||
expect(parsed.images).toHaveLength(0);
|
||||
expect(parsed.offloadedRefs).toHaveLength(0);
|
||||
expect(saveMediaBufferMock).not.toHaveBeenCalled();
|
||||
@@ -468,7 +471,10 @@ describe("parseMessageWithAttachments validation errors", () => {
|
||||
expect(parsed.offloadedRefs).toHaveLength(1);
|
||||
expect(parsed.offloadedRefs[0]?.mimeType).toBe("application/pdf");
|
||||
expect(parsed.offloadedRefs[0]?.label).toBe("brief.pdf");
|
||||
expect(parsed.message).toBe("read this");
|
||||
expect(parsed.message).toBe(
|
||||
`read this\n[media attached: ${parsed.offloadedRefs[0]?.mediaRef}]`,
|
||||
);
|
||||
expect(parsed.messageWithoutOffloadedImageRefs).toBe(parsed.message);
|
||||
} finally {
|
||||
await cleanupOffloadedRefs(parsed.offloadedRefs);
|
||||
}
|
||||
@@ -495,6 +501,7 @@ describe("parseMessageWithAttachments validation errors", () => {
|
||||
const offloaded = expectDefined(parsed.offloadedRefs[0], "offloaded image ref");
|
||||
expect(offloaded.mimeType).toBe("image/png");
|
||||
expect(parsed.message).toBe(`see this\n[media attached: ${offloaded.mediaRef}]`);
|
||||
expect(parsed.messageWithoutOffloadedImageRefs).toBe("see this");
|
||||
expect(parsed.media).toEqual([
|
||||
{
|
||||
path: offloaded.path,
|
||||
@@ -530,6 +537,9 @@ describe("parseMessageWithAttachments validation errors", () => {
|
||||
expect(parsed.message).toContain(
|
||||
"[image attachment omitted: text-only attachment limit reached]",
|
||||
);
|
||||
expect(parsed.messageWithoutOffloadedImageRefs).toBe(
|
||||
"see these\n[image attachment omitted: text-only attachment limit reached]",
|
||||
);
|
||||
expect(logs).toEqual([
|
||||
"attachment dot-10.png: dropping image because text-only offload limit 10 was reached",
|
||||
]);
|
||||
|
||||
@@ -36,6 +36,7 @@ export type OffloadedRef = {
|
||||
|
||||
type ParsedMessageWithImages = {
|
||||
message: string;
|
||||
messageWithoutOffloadedImageRefs: string;
|
||||
images: ChatImageContent[];
|
||||
imageOrder: PromptImageOrderEntry[];
|
||||
media: MediaFact[];
|
||||
@@ -322,13 +323,21 @@ export async function parseMessageWithAttachments(
|
||||
const acceptNonImage = opts?.acceptNonImage !== false;
|
||||
|
||||
if (!attachments || attachments.length === 0) {
|
||||
return { message, images: [], imageOrder: [], media: [], offloadedRefs: [] };
|
||||
return {
|
||||
message,
|
||||
messageWithoutOffloadedImageRefs: message,
|
||||
images: [],
|
||||
imageOrder: [],
|
||||
media: [],
|
||||
offloadedRefs: [],
|
||||
};
|
||||
}
|
||||
|
||||
const images: ChatImageContent[] = [];
|
||||
const imageOrder: PromptImageOrderEntry[] = [];
|
||||
const offloadedRefs: OffloadedRef[] = [];
|
||||
let updatedMessage = message;
|
||||
let messageWithoutOffloadedImageRefs = message;
|
||||
let textOnlyImageOffloadCount = 0;
|
||||
const savedMediaIds: string[] = [];
|
||||
|
||||
@@ -422,6 +431,8 @@ export async function parseMessageWithAttachments(
|
||||
`${TEXT_ONLY_OFFLOAD_LIMIT} was reached`,
|
||||
);
|
||||
updatedMessage += "\n[image attachment omitted: text-only attachment limit reached]";
|
||||
messageWithoutOffloadedImageRefs +=
|
||||
"\n[image attachment omitted: text-only attachment limit reached]";
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -458,8 +469,10 @@ export async function parseMessageWithAttachments(
|
||||
savedMediaIds.push(savedMedia.id);
|
||||
|
||||
const mediaRef = `media://inbound/${savedMedia.id}`;
|
||||
if (isImage) {
|
||||
updatedMessage += `\n[media attached: ${mediaRef}]`;
|
||||
const mediaLine = `\n[media attached: ${mediaRef}]`;
|
||||
updatedMessage += mediaLine;
|
||||
if (!isImage) {
|
||||
messageWithoutOffloadedImageRefs += mediaLine;
|
||||
}
|
||||
log?.info?.(
|
||||
shouldForceImageOffload && isImage
|
||||
@@ -491,6 +504,10 @@ export async function parseMessageWithAttachments(
|
||||
|
||||
return {
|
||||
message: updatedMessage !== message ? updatedMessage.trimEnd() : message,
|
||||
messageWithoutOffloadedImageRefs:
|
||||
messageWithoutOffloadedImageRefs !== message
|
||||
? messageWithoutOffloadedImageRefs.trimEnd()
|
||||
: message,
|
||||
images,
|
||||
imageOrder,
|
||||
media: offloadedRefs.map((ref) => ({
|
||||
|
||||
@@ -57,23 +57,6 @@ function logAttachmentFailure(
|
||||
});
|
||||
}
|
||||
|
||||
function stripTrailingOffloadedMediaMarkers(message: string, refs: OffloadedRef[]): string {
|
||||
if (refs.length === 0) {
|
||||
return message;
|
||||
}
|
||||
const removableRefs = new Set(refs.map((ref) => ref.mediaRef));
|
||||
const lines = message.split(/\r?\n/);
|
||||
while (lines.length > 0) {
|
||||
const last = lines[lines.length - 1]?.trim() ?? "";
|
||||
const match = /^\[media attached:\s*(media:\/\/inbound\/[^\]\s]+)\]$/.exec(last);
|
||||
if (!match?.[1] || !removableRefs.delete(match[1])) {
|
||||
break;
|
||||
}
|
||||
lines.pop();
|
||||
}
|
||||
return lines.join("\n").trimEnd();
|
||||
}
|
||||
|
||||
function isPdfOffloadedRef(ref: OffloadedRef): boolean {
|
||||
const mime = ref.mimeType.trim().toLowerCase();
|
||||
if (mime === "application/pdf" || mime.endsWith("+pdf")) {
|
||||
@@ -263,14 +246,11 @@ export async function prepareChatSendAttachments(params: {
|
||||
supportsImages,
|
||||
acceptNonImage: true,
|
||||
});
|
||||
parsedMessage = stripTrailingOffloadedMediaMarkers(
|
||||
parsed.message,
|
||||
routeImageOffloadsAsMediaPaths
|
||||
? parsed.offloadedRefs.filter((ref) => ref.mimeType.startsWith("image/"))
|
||||
: [],
|
||||
);
|
||||
parsedMessage = routeImageOffloadsAsMediaPaths
|
||||
? parsed.messageWithoutOffloadedImageRefs
|
||||
: parsed.message;
|
||||
parsedImages = parsed.images;
|
||||
imageOrder = routeImageOffloadsAsMediaPaths ? [] : parsed.imageOrder;
|
||||
imageOrder = parsed.imageOrder;
|
||||
offloadedRefs = parsed.offloadedRefs;
|
||||
({
|
||||
paths: mediaPathOffloadPaths,
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
GATEWAY_CLIENT_IDS,
|
||||
GATEWAY_CLIENT_MODES,
|
||||
type GatewayClientInfo,
|
||||
} from "../../../packages/gateway-protocol/src/client-info.js";
|
||||
import { createSolidPngBuffer } from "../../../test/helpers/image-fixtures.js";
|
||||
import { pruneProcessedHistoryImages } from "../../agents/embedded-agent-runner/run/history-image-prune.js";
|
||||
import { hydratePromptMediaMessages } from "../../agents/embedded-agent-runner/run/images.js";
|
||||
import type { AgentMessage } from "../../agents/runtime/index.js";
|
||||
import type { MsgContext } from "../../auto-reply/templating.js";
|
||||
import type { UserTurnInput } from "../../sessions/user-turn-transcript.js";
|
||||
import { resolveStateDir } from "../../config/paths.js";
|
||||
import {
|
||||
buildPersistedUserTurnMessage,
|
||||
type UserTurnInput,
|
||||
} from "../../sessions/user-turn-transcript.js";
|
||||
import { applyChatSendManagedMediaFields, prepareChatSendUserTurn } from "./chat-send-user-turn.js";
|
||||
|
||||
function createUserTurnInputController() {
|
||||
@@ -206,8 +216,8 @@ describe("prepareChatSendUserTurn", () => {
|
||||
await expect(readInput()).resolves.toEqual(controller.baseInput);
|
||||
});
|
||||
|
||||
it("carries retained image claim-check facts without changing the trailing prompt line", () => {
|
||||
const { controller } = createUserTurnInputController();
|
||||
it("carries retained image claim-check facts without changing the trailing prompt line", async () => {
|
||||
const { controller, readInput } = createUserTurnInputController();
|
||||
const mediaRef = "media://inbound/image-1.png";
|
||||
const prepared = prepareChatSendUserTurn({
|
||||
request: {
|
||||
@@ -255,6 +265,166 @@ describe("prepareChatSendUserTurn", () => {
|
||||
contentType: "image/png",
|
||||
},
|
||||
]);
|
||||
await expect(readInput()).resolves.toMatchObject({
|
||||
mediaImageLayout: { slots: [{ kind: "offloaded", factIndex: 0 }] },
|
||||
});
|
||||
});
|
||||
|
||||
it("persists and prunes the staged PDF claim-check alias as structured ownership", async () => {
|
||||
const { controller, readInput } = createUserTurnInputController();
|
||||
const mediaRef = "media://inbound/report.pdf";
|
||||
prepareChatSendUserTurn({
|
||||
request: {
|
||||
clientInfo: createClientInfo(),
|
||||
normalizedAttachments: [{}],
|
||||
suppressCommandInterpretation: false,
|
||||
systemInputProvenance: undefined,
|
||||
systemProvenanceReceipt: undefined,
|
||||
},
|
||||
session: {
|
||||
agentId: "main",
|
||||
clientRunId: "run-1",
|
||||
sessionKey: "agent:main:main",
|
||||
},
|
||||
admission: {
|
||||
originatingRoute: { originatingChannel: "webchat", explicitDeliverRoute: false },
|
||||
},
|
||||
attachments: createAttachments({
|
||||
offloadedRefs: [
|
||||
{
|
||||
mediaRef,
|
||||
id: "report.pdf",
|
||||
path: "/media/inbound/report.pdf",
|
||||
mimeType: "application/pdf",
|
||||
label: "report.pdf",
|
||||
sizeBytes: 10,
|
||||
},
|
||||
],
|
||||
parsedMessage: `read this\n[media attached: ${mediaRef}]`,
|
||||
}),
|
||||
client: null,
|
||||
logGateway: { warn: vi.fn() } as never,
|
||||
userTurn: controller,
|
||||
});
|
||||
|
||||
const input = await readInput();
|
||||
expect(input.media).toEqual([
|
||||
{
|
||||
path: "/media/inbound/report.pdf",
|
||||
url: mediaRef,
|
||||
contentType: "application/pdf",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
]);
|
||||
const persisted = buildPersistedUserTurnMessage({
|
||||
...input,
|
||||
text: `read this\n[media attached: ${mediaRef}]`,
|
||||
});
|
||||
const history = [
|
||||
persisted,
|
||||
{ role: "assistant", content: "ack" },
|
||||
{ role: "user", content: "more" },
|
||||
{ role: "assistant", content: "ack" },
|
||||
{ role: "user", content: "more" },
|
||||
{ role: "assistant", content: "ack" },
|
||||
{ role: "user", content: "more" },
|
||||
{ role: "assistant", content: "ack" },
|
||||
] as unknown as Parameters<typeof pruneProcessedHistoryImages>[0];
|
||||
const pruned = pruneProcessedHistoryImages(history);
|
||||
const first = pruned?.[0] as unknown as Record<string, unknown> | undefined;
|
||||
expect(first?.content).toBe(
|
||||
"read this\n[media reference removed - already processed by model]",
|
||||
);
|
||||
expect((first?.["__openclaw"] as Record<string, unknown> | undefined)?.media).toBeUndefined();
|
||||
});
|
||||
|
||||
it("hydrates and prunes a staged image claim-check alias as structured ownership", async () => {
|
||||
const id = `gateway-image-${Date.now()}-${Math.random().toString(36).slice(2)}.png`;
|
||||
const imagePath = path.join(resolveStateDir(), "media", "inbound", id);
|
||||
const mediaRef = `media://inbound/${id}`;
|
||||
const unownedRef = "media://inbound/unowned.png";
|
||||
const text = `inspect\n[media attached: ${mediaRef}]\n[media attached: ${unownedRef}]`;
|
||||
await fs.mkdir(path.dirname(imagePath), { recursive: true });
|
||||
await fs.writeFile(imagePath, createSolidPngBuffer(2, 2, { r: 10, g: 20, b: 30 }));
|
||||
|
||||
try {
|
||||
const { controller, readInput } = createUserTurnInputController();
|
||||
prepareChatSendUserTurn({
|
||||
request: {
|
||||
clientInfo: createClientInfo(),
|
||||
normalizedAttachments: [{}],
|
||||
suppressCommandInterpretation: false,
|
||||
systemInputProvenance: undefined,
|
||||
systemProvenanceReceipt: undefined,
|
||||
},
|
||||
session: {
|
||||
agentId: "main",
|
||||
clientRunId: "run-1",
|
||||
sessionKey: "agent:main:main",
|
||||
},
|
||||
admission: {
|
||||
originatingRoute: { originatingChannel: "webchat", explicitDeliverRoute: false },
|
||||
},
|
||||
attachments: createAttachments({
|
||||
imageOrder: ["offloaded"],
|
||||
offloadedRefs: [
|
||||
{
|
||||
mediaRef,
|
||||
id,
|
||||
path: imagePath,
|
||||
mimeType: "image/png",
|
||||
label: "image.png",
|
||||
sizeBytes: 10,
|
||||
},
|
||||
],
|
||||
parsedMessage: text,
|
||||
}),
|
||||
client: null,
|
||||
logGateway: { warn: vi.fn() } as never,
|
||||
userTurn: controller,
|
||||
});
|
||||
|
||||
const input = await readInput();
|
||||
expect(input.media).toEqual([{ path: imagePath, url: mediaRef, contentType: "image/png" }]);
|
||||
expect(input.media?.[0]).not.toHaveProperty("hydrationSuppressed");
|
||||
const persisted = buildPersistedUserTurnMessage({ ...input, text });
|
||||
expect(
|
||||
(
|
||||
(persisted as unknown as Record<string, unknown>)["__openclaw"] as {
|
||||
media?: unknown;
|
||||
}
|
||||
).media,
|
||||
).toEqual([{ path: imagePath, url: mediaRef, contentType: "image/png" }]);
|
||||
|
||||
const hydrated = await hydratePromptMediaMessages([persisted as AgentMessage], {
|
||||
workspaceDir: path.dirname(imagePath),
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: true,
|
||||
});
|
||||
expect((hydrated[0] as unknown as { content?: unknown[] }).content).toEqual([
|
||||
{ type: "text", text },
|
||||
expect.objectContaining({ type: "image", mimeType: "image/png" }),
|
||||
]);
|
||||
|
||||
const history = [
|
||||
persisted,
|
||||
{ role: "assistant", content: "ack" },
|
||||
{ role: "user", content: "more" },
|
||||
{ role: "assistant", content: "ack" },
|
||||
{ role: "user", content: "more" },
|
||||
{ role: "assistant", content: "ack" },
|
||||
{ role: "user", content: "more" },
|
||||
{ role: "assistant", content: "ack" },
|
||||
] as unknown as Parameters<typeof pruneProcessedHistoryImages>[0];
|
||||
const pruned = pruneProcessedHistoryImages(history);
|
||||
const first = pruned?.[0] as unknown as Record<string, unknown> | undefined;
|
||||
expect(first?.content).toBe(
|
||||
`inspect\n[media reference removed - already processed by model]\n[media attached: ${unownedRef}]`,
|
||||
);
|
||||
expect((first?.["__openclaw"] as Record<string, unknown> | undefined)?.media).toBeUndefined();
|
||||
} finally {
|
||||
await fs.rm(imagePath, { force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -93,11 +93,26 @@ export function applyChatSendManagedMediaFields(
|
||||
}
|
||||
}
|
||||
|
||||
function buildChatSendUserTurnMedia(savedMedia: SavedMedia[]): NonNullable<UserTurnInput["media"]> {
|
||||
return savedMedia.map((entry) => ({
|
||||
path: entry.path,
|
||||
contentType: entry.contentType,
|
||||
}));
|
||||
function buildChatSendUserTurnMedia(
|
||||
savedMedia: SavedMedia[],
|
||||
offloadedRefs: OffloadedRef[],
|
||||
): NonNullable<UserTurnInput["media"]> {
|
||||
const offloadedRefsById = new Map(offloadedRefs.map((ref) => [ref.id, ref] as const));
|
||||
return savedMedia.map((entry) => {
|
||||
const offloadedRef = offloadedRefsById.get(entry.id);
|
||||
return {
|
||||
path: entry.path,
|
||||
...(offloadedRef
|
||||
? {
|
||||
// Every offload keeps its claim-check alias so persisted marker
|
||||
// ownership survives; only non-images skip native image hydration.
|
||||
url: offloadedRef.mediaRef,
|
||||
...(offloadedRef.mimeType.startsWith("image/") ? {} : { hydrationSuppressed: true }),
|
||||
}
|
||||
: {}),
|
||||
contentType: entry.contentType,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function buildChatSendPromptMedia(
|
||||
@@ -251,10 +266,21 @@ export function prepareChatSendUserTurn(params: {
|
||||
? getPersistedMediaForTranscript()
|
||||
: Promise.resolve([]);
|
||||
userTurn.setInputPromise(
|
||||
preparedUserTurnMediaPromise.then(buildChatSendUserTurnMedia).then((media) => ({
|
||||
...userTurn.baseInput,
|
||||
...(media.length > 0 ? { media } : {}),
|
||||
})),
|
||||
preparedUserTurnMediaPromise
|
||||
.then((media) => buildChatSendUserTurnMedia(media, attachments.offloadedRefs))
|
||||
.then((media) => ({
|
||||
...userTurn.baseInput,
|
||||
...(media.length > 0 ? { media } : {}),
|
||||
...(media.length > 0 && attachments.imageOrder.length > 0
|
||||
? {
|
||||
mediaImageLayout: {
|
||||
// persistInboundImagesForTranscript emits image facts in this exact order,
|
||||
// then appends non-images, so image slot ordinals are fact ordinals.
|
||||
slots: attachments.imageOrder.map((kind, factIndex) => ({ kind, factIndex })),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
})),
|
||||
);
|
||||
const pluginBoundMediaFieldsPromise =
|
||||
attachments.explicitOriginTargetsPlugin && attachments.parsedImages.length > 0
|
||||
|
||||
@@ -5256,7 +5256,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
|
||||
| undefined;
|
||||
expect(mockState.lastDispatchImages).toBeUndefined();
|
||||
expect(mockState.lastDispatchImageOrder).toBeUndefined();
|
||||
expect(mockState.lastDispatchCtx?.Body).toBe("summarize this");
|
||||
expect(mockState.lastDispatchCtx?.Body).toBe(
|
||||
"summarize this\n[media attached: media://inbound/saved-media]",
|
||||
);
|
||||
expect(mockState.savedMediaCalls[0]?.contentType).toBe("application/pdf");
|
||||
expect(mockState.savedMediaCalls[0]?.subdir).toBe("inbound");
|
||||
expect(typeof mockState.savedMediaCalls[0]?.size).toBe("number");
|
||||
@@ -5635,7 +5637,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
|
||||
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
|
||||
});
|
||||
expect(mockState.lastDispatchImages).toBeUndefined();
|
||||
expect(mockState.lastDispatchImageOrder).toBeUndefined();
|
||||
expect(mockState.lastDispatchImageOrder).toEqual(["offloaded"]);
|
||||
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
|
||||
expect(mockState.lastDispatchCtx?.Body).not.toContain("media://");
|
||||
expect(mockState.lastDispatchCtx?.MediaPath).toBe("/tmp/1.png");
|
||||
@@ -5811,7 +5813,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
|
||||
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
|
||||
});
|
||||
expect(mockState.lastDispatchImages).toBeUndefined();
|
||||
expect(mockState.lastDispatchImageOrder).toBeUndefined();
|
||||
expect(mockState.lastDispatchImageOrder).toEqual(["offloaded"]);
|
||||
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
|
||||
expect(mockState.lastDispatchCtx?.Body).not.toContain("media://");
|
||||
expect(mockState.lastDispatchCtx?.MediaPath).toBe("/tmp/1.png");
|
||||
@@ -5876,11 +5878,14 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
|
||||
);
|
||||
expect(mockState.lastDispatchCtx?.MediaTypes).toEqual(["application/pdf"]);
|
||||
expect(mockState.lastDispatchCtx?.MediaType).toBe("application/pdf");
|
||||
// Non-image offloads MUST NOT inject a media://URI into the prompt body —
|
||||
// they ride through ctx.MediaPaths so buildInboundMediaNote prepends the
|
||||
// real path, avoiding duplicate media markers.
|
||||
expect(mockState.lastDispatchCtx?.Body).not.toContain("media://");
|
||||
expect(mockState.lastDispatchCtx?.BodyForAgent).not.toContain("media://");
|
||||
// Non-image offloads retain their claim-check line while the staged path
|
||||
// also travels structurally for media tools and transcript persistence.
|
||||
expect(mockState.lastDispatchCtx?.Body).toContain(
|
||||
"[media attached: media://inbound/saved-media]",
|
||||
);
|
||||
expect(mockState.lastDispatchCtx?.BodyForAgent).toContain(
|
||||
"[media attached: media://inbound/saved-media]",
|
||||
);
|
||||
expect(mockState.lastDispatchImages).toBeUndefined();
|
||||
// Marker replaces the implicit "relative-path no-op" coupling in
|
||||
// get-reply.ts with an explicit skip contract.
|
||||
@@ -5939,7 +5944,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
|
||||
]);
|
||||
expect(mockState.lastDispatchCtx?.MediaTypes).toEqual(["application/zip"]);
|
||||
expect(mockState.lastDispatchImages).toBeUndefined();
|
||||
expect(mockState.lastDispatchCtx?.Body).not.toContain("media://");
|
||||
expect(mockState.lastDispatchCtx?.Body).toContain(
|
||||
"[media attached: media://inbound/saved-media]",
|
||||
);
|
||||
expect(mockState.lastDispatchCtx?.MediaStaged).toBe(true);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { MediaKind } from "@openclaw/media-core/constants";
|
||||
import { kindFromMime } from "@openclaw/media-core/mime";
|
||||
import { kindFromMime, mimeTypeFromFilePath } from "@openclaw/media-core/mime";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import type { PromptImageOrderEntry } from "./prompt-image-order.js";
|
||||
|
||||
/** One ordered runtime attachment; array position is its alignment identity. */
|
||||
export type MediaFact = {
|
||||
@@ -11,6 +12,10 @@ export type MediaFact = {
|
||||
transcribed?: boolean;
|
||||
messageId?: string;
|
||||
workspaceDir?: string;
|
||||
// Declared field, not a symbol: suppression must survive every fact copy or
|
||||
// reprojection boundary; described images otherwise rehydrate or count failed.
|
||||
// Structured persistence may retain it; legacy Media* projections never emit it.
|
||||
hydrationSuppressed?: boolean;
|
||||
};
|
||||
|
||||
export type MediaFactInput = {
|
||||
@@ -23,14 +28,66 @@ const RUNTIME_PROMPT_MEDIA_FACTS = Symbol.for("openclaw.runtimePromptMediaFacts"
|
||||
export function attachRuntimePromptMediaFacts<T extends object>(
|
||||
message: T,
|
||||
media: readonly MediaFact[],
|
||||
imageOrder?: readonly PromptImageOrderEntry[],
|
||||
): T {
|
||||
const normalized = normalizeMediaFacts(media);
|
||||
if (imageOrder?.length) {
|
||||
Object.defineProperty(normalized, "imageOrder", { value: [...imageOrder] });
|
||||
}
|
||||
Object.defineProperty(message, RUNTIME_PROMPT_MEDIA_FACTS, {
|
||||
configurable: true,
|
||||
value: normalizeMediaFacts(media),
|
||||
value: normalized,
|
||||
});
|
||||
return message;
|
||||
}
|
||||
|
||||
export function readRuntimePromptMediaFacts(message: object): MediaFact[] | undefined {
|
||||
const media = (message as Record<PropertyKey, unknown>)[RUNTIME_PROMPT_MEDIA_FACTS];
|
||||
return Array.isArray(media) ? (media as MediaFact[]) : undefined;
|
||||
}
|
||||
|
||||
export function readRuntimePromptImageOrder(message: object): PromptImageOrderEntry[] | undefined {
|
||||
const imageOrder = (
|
||||
readRuntimePromptMediaFacts(message) as
|
||||
| (MediaFact[] & { imageOrder?: PromptImageOrderEntry[] })
|
||||
| undefined
|
||||
)?.imageOrder;
|
||||
return Array.isArray(imageOrder) ? (imageOrder as PromptImageOrderEntry[]) : undefined;
|
||||
}
|
||||
|
||||
/** Returns whether a fact can produce native image input. */
|
||||
export function isImageMediaFact(fact: MediaFactInput): boolean {
|
||||
if (fact.kind && fact.kind !== "unknown") {
|
||||
return fact.kind === "image" || fact.kind === "sticker";
|
||||
}
|
||||
const contentType = normalizeOptionalString(fact.contentType);
|
||||
const normalizedContentType = contentType?.split(";")[0]?.trim().toLowerCase();
|
||||
if (
|
||||
normalizedContentType &&
|
||||
normalizedContentType !== "application/octet-stream" &&
|
||||
normalizedContentType !== "binary/octet-stream"
|
||||
) {
|
||||
const mimeKind = kindFromMime(normalizedContentType);
|
||||
if (mimeKind) {
|
||||
return mimeKind === "image";
|
||||
}
|
||||
// Legacy channel-mode projections persist bare kinds as MediaType; honor
|
||||
// them, and fall through to filename inference for other unknown strings.
|
||||
if (normalizedContentType === "image" || normalizedContentType === "sticker") {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
normalizedContentType === "audio" ||
|
||||
normalizedContentType === "video" ||
|
||||
normalizedContentType === "document"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const pathValue = normalizeOptionalString(fact.path) ?? normalizeOptionalString(fact.url);
|
||||
return kindFromMime(mimeTypeFromFilePath(pathValue)) === "image";
|
||||
}
|
||||
|
||||
type MediaFactDefaults<TInput extends MediaFactInput = MediaFactInput> = {
|
||||
kind?: MediaKind;
|
||||
messageId?: string;
|
||||
@@ -61,7 +118,7 @@ function normalizeMediaFact<TInput extends MediaFactInput>(
|
||||
): MediaFact {
|
||||
const workspaceDir = normalizeOptionalString(media.workspaceDir) ?? defaults.workspaceDir;
|
||||
const contentType = normalizeOptionalString(media.contentType);
|
||||
return {
|
||||
const normalized: MediaFact = {
|
||||
path: normalizeOptionalString(media.path),
|
||||
url: normalizeOptionalString(media.url),
|
||||
contentType,
|
||||
@@ -69,7 +126,9 @@ function normalizeMediaFact<TInput extends MediaFactInput>(
|
||||
transcribed: media.transcribed === true || defaults.transcribed?.(media, index) === true,
|
||||
messageId: normalizeOptionalString(media.messageId) ?? defaults.messageId,
|
||||
...(workspaceDir ? { workspaceDir } : {}),
|
||||
...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}),
|
||||
};
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/** True when a consumer must use the already-staged legacy path projection. */
|
||||
@@ -136,6 +195,7 @@ export function resolveMediaFacts(source: MediaFactSource): MediaFact[] {
|
||||
workspaceDir:
|
||||
normalizeOptionalString(fact?.workspaceDir) ??
|
||||
normalizeOptionalString(source.MediaWorkspaceDir),
|
||||
hydrationSuppressed: fact?.hydrationSuppressed,
|
||||
},
|
||||
index,
|
||||
);
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
const RUNTIME_PROMPT_IMAGE_FACT_INDEXES = Symbol.for("openclaw.runtimePromptImageFactIndexes");
|
||||
|
||||
type RuntimePromptImageFactIndex = number | null;
|
||||
|
||||
export function finalizeRuntimePromptImages<TImage extends object>(
|
||||
entries: readonly { image: TImage; factIndex: RuntimePromptImageFactIndex }[],
|
||||
): { images: TImage[]; imageFactIndexes: RuntimePromptImageFactIndex[] } {
|
||||
const images = entries.map((entry) => entry.image);
|
||||
const imageFactIndexes = entries.map((entry) => entry.factIndex);
|
||||
attachRuntimePromptImageFactIndexes(images, imageFactIndexes);
|
||||
return { images, imageFactIndexes };
|
||||
}
|
||||
|
||||
/** Carries fact ownership on image blocks without changing provider-visible bytes. */
|
||||
function attachRuntimePromptImageFactIndexes(
|
||||
images: readonly object[],
|
||||
factIndexes: readonly RuntimePromptImageFactIndex[],
|
||||
): void {
|
||||
if (images.length !== factIndexes.length) {
|
||||
return;
|
||||
}
|
||||
Object.defineProperty(images, RUNTIME_PROMPT_IMAGE_FACT_INDEXES, {
|
||||
configurable: true,
|
||||
value: [...factIndexes],
|
||||
});
|
||||
}
|
||||
|
||||
export function readRuntimePromptImageFactIndexes(
|
||||
images: readonly object[] | null | undefined,
|
||||
): RuntimePromptImageFactIndex[] | undefined {
|
||||
if (!images?.length) {
|
||||
return undefined;
|
||||
}
|
||||
const factIndexes = (images as unknown as Record<PropertyKey, unknown>)[
|
||||
RUNTIME_PROMPT_IMAGE_FACT_INDEXES
|
||||
];
|
||||
return Array.isArray(factIndexes) &&
|
||||
factIndexes.length === images.length &&
|
||||
factIndexes.every(
|
||||
(entry) =>
|
||||
entry === null || (typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0),
|
||||
)
|
||||
? (factIndexes as RuntimePromptImageFactIndex[])
|
||||
: undefined;
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import path from "node:path";
|
||||
import { mimeTypeFromFilePath } from "@openclaw/media-core/mime";
|
||||
import type { MediaFactInput } from "../media/media-facts.js";
|
||||
import type { PersistedUserTurnMediaInput } from "./user-turn-transcript.types.js";
|
||||
|
||||
const URL_LIKE_MEDIA_PATH_PATTERN = /^[a-z][a-z0-9+.-]*:/i;
|
||||
const STRUCTURED_MEDIA_KINDS = new Set<NonNullable<MediaFactInput["kind"]>>([
|
||||
"image",
|
||||
"audio",
|
||||
"video",
|
||||
"document",
|
||||
"sticker",
|
||||
"unknown",
|
||||
]);
|
||||
|
||||
function normalizeOptionalText(value: string | null | undefined): string | undefined {
|
||||
const normalized = value?.trim();
|
||||
return normalized ? normalized : undefined;
|
||||
}
|
||||
|
||||
function mediaTypeForTranscript(media: PersistedUserTurnMediaInput, mediaPath?: string): string {
|
||||
return (
|
||||
normalizeOptionalText(media.contentType) ??
|
||||
normalizeOptionalText(media.kind) ??
|
||||
mimeTypeFromFilePath(mediaPath) ??
|
||||
"application/octet-stream"
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeStructuredMediaKind(value: string | null | undefined): MediaFactInput["kind"] {
|
||||
const kind = normalizeOptionalText(value);
|
||||
return kind && STRUCTURED_MEDIA_KINDS.has(kind as NonNullable<MediaFactInput["kind"]>)
|
||||
? (kind as NonNullable<MediaFactInput["kind"]>)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function resolveTranscriptMediaPath(
|
||||
pathValue: string,
|
||||
workspaceDir: string | undefined,
|
||||
): string {
|
||||
// Relative staged media paths are anchored to the media workspace; absolute
|
||||
// paths and URL-like refs are already stable transcript references.
|
||||
if (!workspaceDir || path.isAbsolute(pathValue) || URL_LIKE_MEDIA_PATH_PATTERN.test(pathValue)) {
|
||||
return pathValue;
|
||||
}
|
||||
return path.join(workspaceDir, pathValue);
|
||||
}
|
||||
|
||||
export function normalizeMediaEntryForTranscript(
|
||||
media: PersistedUserTurnMediaInput,
|
||||
): MediaFactInput {
|
||||
const rawPath = normalizeOptionalText(media.path) ?? normalizeOptionalText(media.url);
|
||||
if (!rawPath) {
|
||||
return media.hydrationSuppressed === true
|
||||
? {
|
||||
contentType: normalizeOptionalText(media.contentType),
|
||||
hydrationSuppressed: true,
|
||||
}
|
||||
: {};
|
||||
}
|
||||
return {
|
||||
path: resolveTranscriptMediaPath(rawPath, normalizeOptionalText(media.workspaceDir)),
|
||||
contentType: mediaTypeForTranscript(media, rawPath),
|
||||
...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeStructuredMediaEntryForTranscript(
|
||||
media: PersistedUserTurnMediaInput,
|
||||
): MediaFactInput {
|
||||
const mediaPath = normalizeOptionalText(media.path);
|
||||
const mediaUrl = normalizeOptionalText(media.url);
|
||||
return {
|
||||
path: mediaPath,
|
||||
url: mediaUrl,
|
||||
contentType:
|
||||
normalizeOptionalText(media.contentType) ?? mimeTypeFromFilePath(mediaPath ?? mediaUrl),
|
||||
kind: normalizeStructuredMediaKind(media.kind),
|
||||
workspaceDir: normalizeOptionalText(media.workspaceDir),
|
||||
...(media.hydrationSuppressed === true ? { hydrationSuppressed: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function shouldPersistStructuredMediaEntries(
|
||||
media: readonly PersistedUserTurnMediaInput[] | null | undefined,
|
||||
): boolean {
|
||||
return (media ?? []).some((entry) => {
|
||||
const legacy = normalizeMediaEntryForTranscript(entry);
|
||||
const structured = normalizeStructuredMediaEntryForTranscript(entry);
|
||||
const structuredIdentity = structured.path ?? structured.url;
|
||||
return (
|
||||
structured.hydrationSuppressed === true ||
|
||||
structuredIdentity !== legacy.path ||
|
||||
Boolean(structured.url && structured.url !== legacy.path)
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -1,18 +1,18 @@
|
||||
// User turn transcript helpers extract user-turn text from session transcripts.
|
||||
import path from "node:path";
|
||||
import { mimeTypeFromFilePath } from "@openclaw/media-core/mime";
|
||||
import type { AgentMessage } from "../../packages/agent-core/src/types.js";
|
||||
import {
|
||||
persistSessionTranscriptTurn,
|
||||
type SessionTranscriptTurnPersistOptions,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import {
|
||||
projectMediaFacts,
|
||||
resolveMediaFacts,
|
||||
type MediaFact,
|
||||
type MediaFactInput,
|
||||
} from "../media/media-facts.js";
|
||||
import { projectMediaFacts, resolveMediaFacts, type MediaFact } from "../media/media-facts.js";
|
||||
import { applyInputProvenanceToUserMessage, normalizeInputProvenance } from "./input-provenance.js";
|
||||
import {
|
||||
normalizeMediaEntryForTranscript,
|
||||
normalizeStructuredMediaEntryForTranscript,
|
||||
resolveTranscriptMediaPath,
|
||||
shouldPersistStructuredMediaEntries,
|
||||
} from "./user-turn-transcript.media-normalize.js";
|
||||
import type {
|
||||
CreateUserTurnTranscriptRecorderParams,
|
||||
PersistUserTurnTranscriptParams,
|
||||
@@ -72,37 +72,6 @@ export function resolvePersistedUserTurnText(value: string | null | undefined):
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function mediaTypeForTranscript(media: PersistedUserTurnMediaInput, mediaPath?: string): string {
|
||||
return (
|
||||
normalizeOptionalText(media.contentType) ??
|
||||
normalizeOptionalText(media.kind) ??
|
||||
mimeTypeFromFilePath(mediaPath) ??
|
||||
"application/octet-stream"
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeMediaEntryForTranscript(media: PersistedUserTurnMediaInput): MediaFactInput {
|
||||
const rawPath = normalizeOptionalText(media.path) ?? normalizeOptionalText(media.url);
|
||||
if (!rawPath) {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
path: resolveTranscriptMediaPath(rawPath, normalizeOptionalText(media.workspaceDir)),
|
||||
contentType: mediaTypeForTranscript(media, rawPath),
|
||||
};
|
||||
}
|
||||
|
||||
const URL_LIKE_MEDIA_PATH_PATTERN = /^[a-z][a-z0-9+.-]*:/i;
|
||||
|
||||
function resolveTranscriptMediaPath(pathValue: string, workspaceDir: string | undefined): string {
|
||||
// Relative staged media paths are anchored to the media workspace; absolute
|
||||
// paths and URL-like refs are already stable transcript references.
|
||||
if (!workspaceDir || path.isAbsolute(pathValue) || URL_LIKE_MEDIA_PATH_PATTERN.test(pathValue)) {
|
||||
return pathValue;
|
||||
}
|
||||
return path.join(workspaceDir, pathValue);
|
||||
}
|
||||
|
||||
function resolveTranscriptMediaType(params: {
|
||||
explicitType: string | undefined;
|
||||
mediaPath: string | undefined;
|
||||
@@ -211,6 +180,7 @@ function readOpenClawMessageMeta(message: AgentMessage): Record<string, unknown>
|
||||
|
||||
export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedUserTurnMessage {
|
||||
const mediaFields = buildPersistedUserTurnMediaFields(params.media);
|
||||
const normalizedMedia = (params.media ?? []).map(normalizeStructuredMediaEntryForTranscript);
|
||||
const text = normalizeTranscriptText(params.text);
|
||||
// Storage is BARE (no timestamp prefix). The per-message timestamp is added
|
||||
// at the single LLM-boundary stamping site (normalizeMessagesForLlmBoundary),
|
||||
@@ -223,6 +193,19 @@ export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedU
|
||||
...(params.senderIsOwner === undefined ? {} : { senderIsOwner: params.senderIsOwner }),
|
||||
...senderMeta,
|
||||
...(params.transport ? { transport: params.transport } : {}),
|
||||
...(shouldPersistStructuredMediaEntries(params.media) ? { media: normalizedMedia } : {}),
|
||||
...(params.mediaImageLayout
|
||||
? {
|
||||
mediaImageLayout: {
|
||||
slots: params.mediaImageLayout.slots.map((slot) => ({ ...slot })),
|
||||
...(params.mediaImageLayout.suppressedFactIndexes?.length
|
||||
? {
|
||||
suppressedFactIndexes: [...params.mediaImageLayout.suppressedFactIndexes],
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const message = {
|
||||
role: "user",
|
||||
|
||||
@@ -14,7 +14,10 @@ type UserTurnSessionEntry = {
|
||||
threadId?: string | number;
|
||||
} & Record<string, unknown>;
|
||||
|
||||
export type PersistedUserTurnMediaInput = Pick<MediaFactInput, "contentType" | "path" | "url"> & {
|
||||
export type PersistedUserTurnMediaInput = Pick<
|
||||
MediaFactInput,
|
||||
"contentType" | "hydrationSuppressed" | "path" | "url"
|
||||
> & {
|
||||
kind?: string | null;
|
||||
workspaceDir?: string | null;
|
||||
};
|
||||
@@ -24,6 +27,14 @@ export type PersistedUserTurnMessage = Extract<AgentMessage, { role: "user" }>;
|
||||
export type UserTurnInput = {
|
||||
text?: string | null;
|
||||
media?: readonly PersistedUserTurnMediaInput[] | null;
|
||||
/** Restart-safe native image placement; model-visible prompt bytes remain separate. */
|
||||
mediaImageLayout?: {
|
||||
slots: readonly {
|
||||
kind: "inline" | "offloaded";
|
||||
factIndex?: number;
|
||||
}[];
|
||||
suppressedFactIndexes?: readonly number[];
|
||||
} | null;
|
||||
timestamp?: number;
|
||||
idempotencyKey?: string;
|
||||
senderIsOwner?: boolean;
|
||||
|
||||
Reference in New Issue
Block a user