fix(agents): retain images within tool-result context limits (#120808)

src/agents/embedded-agent-runner/tool-result-context-guard.image.test.ts

@@ -0,0 +1,225 @@
+// Real native image results must retain a bounded, truthful visual context.
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { expectDefined } from "@openclaw/normalization-core";
+import { describe, expect, it } from "vitest";
+import type { OpenClawConfig } from "../../config/types.openclaw.js";
+import { encodePngRgba, fillPixel } from "../../media/png-encode.js";
+import type { AgentMessage } from "../runtime/index.js";
+import { castAgentMessage } from "../test-helpers/agent-message-fixtures.js";
+import { createImageTool } from "../tools/image-tool.js";
+import {
+  createMessageCharEstimateCache,
+  estimateMessageCharsCached,
+  TOOL_RESULT_CHARS_PER_TOKEN_ESTIMATE,
+} from "./tool-result-char-estimator.js";
+import { installToolResultContextGuard } from "./tool-result-context-guard.js";
+
+type ContentBlock = { type: string; text?: string; data?: string; mimeType?: string };
+
+function createRequiredImageTool() {
+  const config: OpenClawConfig = {
+    agents: { defaults: { imageMaxDimensionPx: 32 } },
+  };
+  return expectDefined(
+    createImageTool({
+      config,
+      agentDir: join(tmpdir(), "openclaw-native-image-context-test"),
+      modelHasVision: true,
+    }),
+    "native-vision image tool is available",
+  );
+}
+
+function makeDistinctImageRef(index: number): string {
+  const width = 64;
+  const pixels = Buffer.alloc(width * width * 4);
+  for (let y = 0; y < width; y += 1) {
+    for (let x = 0; x < width; x += 1) {
+      fillPixel(pixels, x, y, width, (index + 1) * 17, x * 3, y * 3);
+    }
+  }
+  return `data:image/png;base64,${encodePngRgba(pixels, width, width).toString("base64")}`;
+}
+
+async function executeNativeImageTool(imageCount: number): Promise<AgentMessage> {
+  const result = await createRequiredImageTool().execute("native-image-context", {
+    images: Array.from({ length: imageCount }, (_, index) => makeDistinctImageRef(index)),
+  });
+  const content = result.content as ContentBlock[];
+  expect(content[0]?.text).toBe(
+    `Loaded ${imageCount} image${imageCount === 1 ? "" : "s"} for direct visual inspection.`,
+  );
+  expect(content.filter((block) => block.type === "image")).toHaveLength(imageCount);
+  expect(content.filter((block) => block.type === "image")).toEqual(
+    expect.arrayContaining([expect.objectContaining({ mimeType: "image/jpeg" })]),
+  );
+  expect(result.details).toMatchObject({ transport: "native", media: { outbound: false } });
+  return castAgentMessage({
+    role: "toolResult",
+    toolCallId: "native-image-context",
+    toolName: "image",
+    isError: false,
+    timestamp: 0,
+    ...result,
+  });
+}
+
+async function projectForContext(
+  source: AgentMessage,
+  contextWindowTokens: number,
+): Promise<AgentMessage> {
+  const agent: {
+    transformContext?: (messages: AgentMessage[], signal: AbortSignal) => Promise<AgentMessage[]>;
+  } = {};
+  installToolResultContextGuard({ agent, contextWindowTokens });
+  const transformed = await expectDefined(agent.transformContext, "installed context guard")(
+    [source],
+    new AbortController().signal,
+  );
+  return expectDefined(transformed[0], "guarded native image result");
+}
+
+function blocksOf(message: AgentMessage): ContentBlock[] {
+  return (message as { content: ContentBlock[] }).content;
+}
+
+function expectWithinExistingCap(message: AgentMessage, contextWindowTokens: number): void {
+  const existingCap = Math.max(
+    1_024,
+    Math.floor(contextWindowTokens * TOOL_RESULT_CHARS_PER_TOKEN_ESTIMATE * 0.5),
+  );
+  expect(estimateMessageCharsCached(message, createMessageCharEstimateCache())).toBeLessThanOrEqual(
+    existingCap,
+  );
+}
+
+describe("native image tool result context projection", () => {
+  it.each([
+    { contextWindowTokens: 8_000, loadedImages: 1, visibleImages: 0 },
+    { contextWindowTokens: 32_000, loadedImages: 3, visibleImages: 1 },
+    { contextWindowTokens: 128_000, loadedImages: 9, visibleImages: 7 },
+  ])(
+    "preserves a fitting sanitized image prefix in a $contextWindowTokens-token window",
+    async ({ contextWindowTokens, loadedImages, visibleImages }) => {
+      const source = await executeNativeImageTool(loadedImages);
+      const originalSnapshot = structuredClone(source);
+      const projected = await projectForContext(source, contextWindowTokens);
+      const sourceImages = blocksOf(source).filter((block) => block.type === "image");
+      const projectedImages = blocksOf(projected).filter((block) => block.type === "image");
+      const projectedText = blocksOf(projected)
+        .filter((block) => block.type === "text")
+        .map((block) => block.text)
+        .join("\n");
+
+      expect(projectedImages).toEqual(sourceImages.slice(0, visibleImages));
+      expect(projectedText).toMatch(
+        new RegExp(
+          `${loadedImages - visibleImages} images? .*omitted|omitted.*${loadedImages - visibleImages} images?`,
+          "i",
+        ),
+      );
+      expect(projectedText).toMatch(/context/i);
+      if (visibleImages === 0) {
+        expect(projectedText).toMatch(
+          /no images? (?:fit|included|available)|images? (?:could not|cannot) (?:fit|be included)/i,
+        );
+      }
+      expectWithinExistingCap(projected, contextWindowTokens);
+      expect(source).toEqual(originalSnapshot);
+      expect((source as { details?: unknown }).details).toMatchObject({
+        media: { outbound: false },
+      });
+      expect(projected).not.toHaveProperty("details");
+    },
+  );
+
+  it("leaves sanitized native images and private details untouched when all blocks fit", async () => {
+    const source = await executeNativeImageTool(2);
+    const projected = await projectForContext(source, 32_768);
+
+    expect(projected).toBe(source);
+    expect(blocksOf(projected).filter((block) => block.type === "image")).toHaveLength(2);
+    expect((projected as { details?: unknown }).details).toEqual(
+      (source as { details?: unknown }).details,
+    );
+    expectWithinExistingCap(projected, 32_768);
+  });
+
+  it("preserves mixed text, unknown blocks, CJK content, image order, and original privacy", async () => {
+    const native = await executeNativeImageTool(3);
+    const nativeBlocks = blocksOf(native);
+    const unknown = { type: "custom", value: { label: "bounded metadata" } };
+    const cjk = { type: "text", text: "画像の順序を維持する 🖼️" };
+    const source = castAgentMessage({
+      ...native,
+      toolName: "read",
+      content: [nativeBlocks[0], nativeBlocks[1], unknown, cjk, nativeBlocks[2], nativeBlocks[3]],
+      details: { media: { outbound: false }, privateReference: "private-fixture-reference" },
+    });
+    const projected = await projectForContext(source, 32_000);
+    const content = blocksOf(projected);
+
+    expect(content.filter((block) => block.type === "image")).toEqual([nativeBlocks[1]]);
+    expect(content.map((block) => block.type)).toEqual(["text", "image", "custom", "text", "text"]);
+    expect(content).toContain(unknown);
+    expect(content).toContain(cjk);
+    expect(content.at(-1)?.text).toMatch(/2 images? .*omitted|omitted.*2 images?/i);
+    expectWithinExistingCap(projected, 32_000);
+    expect(projected).not.toHaveProperty("details");
+    expect(JSON.stringify(blocksOf(projected))).not.toContain("private-fixture-reference");
+    expect(
+      (source as { details?: { media?: { outbound?: boolean } } }).details?.media?.outbound,
+    ).toBe(false);
+  });
+
+  it.each([1, 2])(
+    "retains a fitting image when oversized CJK text accompanies %i sanitized images",
+    async (loadedImages) => {
+      const native = await executeNativeImageTool(loadedImages);
+      const nativeBlocks = blocksOf(native);
+      const source = castAgentMessage({
+        ...native,
+        content: [{ type: "text", text: "画像🖼️".repeat(8_000) }, ...nativeBlocks.slice(1)],
+      });
+      const projected = await projectForContext(source, 32_000);
+      const content = blocksOf(projected);
+
+      expect(content.filter((block) => block.type === "image")).toEqual([nativeBlocks[1]]);
+      expect(content[0]?.text).toContain("画像");
+      const boundedText = expectDefined(content[0]?.text, "bounded image context text");
+      expect(Buffer.from(boundedText, "utf8").toString("utf8")).toBe(boundedText);
+      expectWithinExistingCap(projected, 32_000);
+      if (loadedImages === 1) {
+        expect(content.some((block) => block.text?.includes("0 images"))).toBe(false);
+      } else {
+        expect(content.some((block) => block.text?.includes("1 image omitted"))).toBe(true);
+      }
+      expect(blocksOf(source).filter((block) => block.type === "image")).toHaveLength(loadedImages);
+    },
+  );
+
+  it("keeps a fitting image without letting oversized text starve a later text block", async () => {
+    const native = await executeNativeImageTool(2);
+    const nativeBlocks = blocksOf(native);
+    const trailingText = { type: "text", text: "retain the image description" };
+    const source = castAgentMessage({
+      ...native,
+      content: [
+        { type: "text", text: "画像🖼️".repeat(8_000) },
+        nativeBlocks[1],
+        trailingText,
+        nativeBlocks[2],
+      ],
+    });
+    const projected = await projectForContext(source, 32_000);
+    const content = blocksOf(projected);
+
+    expect(content.filter((block) => block.type === "image")).toEqual([nativeBlocks[1]]);
+    const boundedText = expectDefined(content[0]?.text, "bounded multi-block image context text");
+    expect(Buffer.from(boundedText, "utf8").toString("utf8")).toBe(boundedText);
+    expect(content).toContainEqual(trailingText);
+    expect(content.at(-1)?.text).toContain("1 image omitted");
+    expectWithinExistingCap(projected, 32_000);
+  });
+});

src/agents/embedded-agent-runner/tool-result-context-guard.ts

@@ -20,7 +20,7 @@ import {
   getToolResultText,
   isToolResultMessage,
 } from "./tool-result-char-estimator.js";
-import { truncateToolResultText } from "./tool-result-truncation.js";
+import { truncateToolResultMessage, truncateToolResultText } from "./tool-result-truncation.js";
 
 const SINGLE_TOOL_RESULT_CONTEXT_SHARE = 0.5;
 const TRANSCRIPT_PROMPT_TEXT_KEY = "__openclawTranscriptPromptText";
@@ -130,16 +130,19 @@ function stripTranscriptPromptMarkers(messages: AgentMessage[]): AgentMessage[]
   return changed ? stripped : messages;
 }
 
-function replaceToolResultText(msg: AgentMessage, text: string): AgentMessage {
+function replaceToolResultContent(
+  msg: AgentMessage,
+  replacement: string | unknown[],
+): AgentMessage {
   const content = (msg as { content?: unknown }).content;
-  const replacementContent =
-    typeof content === "string" || content === undefined ? text : [{ type: "text", text }];
-
   const sourceRecord = msg as unknown as Record<string, unknown>;
   const { details: _details, ...rest } = sourceRecord;
   return {
     ...rest,
-    content: replacementContent,
+    content:
+      typeof replacement === "string" && !(typeof content === "string" || content === undefined)
+        ? [{ type: "text", text: replacement }]
+        : replacement,
   } as AgentMessage;
 }
 
@@ -160,18 +163,93 @@ function truncateToolResultToChars(
   if (estimatedChars <= maxChars) {
     return msg;
   }
+  let rawText = getToolResultText(msg);
+  const content = (msg as { content?: unknown }).content;
+  if (Array.isArray(content)) {
+    const isImage = (block: unknown) =>
+      Boolean(block) && typeof block === "object" && (block as { type?: unknown }).type === "image";
+    const isText = (block: unknown): block is { type: "text"; text: string } =>
+      Boolean(block) &&
+      typeof block === "object" &&
+      (block as { type?: unknown }).type === "text" &&
+      typeof (block as { text?: unknown }).text === "string";
+    const imageCount = content.filter(isImage).length;
+    if (imageCount > 0) {
+      const omissionNotice = (retainedImages: number) => {
+        const omittedImages = imageCount - retainedImages;
+        return (
+          `[${omittedImages} image${omittedImages === 1 ? "" : "s"} omitted from context` +
+          `${retainedImages === 0 ? "; no images fit the context limit" : ""}; rerun with fewer images]`
+        );
+      };
+
+      // Reserve images first; shared multi-block truncation preserves later text
+      // while the existing weighted estimator enforces the unchanged hard cap.
+      for (let retainedImages = imageCount; retainedImages >= 0; retainedImages -= 1) {
+        let seenImages = 0;
+        const retainedContent = content.filter(
+          (block) => !isImage(block) || ++seenImages <= retainedImages,
+        );
+        const notice =
+          retainedImages < imageCount
+            ? [{ type: "text", text: omissionNotice(retainedImages) }]
+            : [];
+        const reservedChars = estimateMessageCharsCached(
+          replaceToolResultContent(msg, [
+            ...retainedContent.filter((block) => !isText(block)),
+            ...notice,
+          ]),
+          cache,
+        );
+        const availableTextChars = maxChars - reservedChars;
+        if (availableTextChars < 0 || (availableTextChars === 0 && retainedContent.some(isText))) {
+          continue;
+        }
+        let textBudget = availableTextChars;
+        while (true) {
+          const bounded = truncateToolResultMessage(
+            replaceToolResultContent(msg, retainedContent),
+            textBudget,
+          );
+          const projectedContent = (bounded as { content: unknown[] }).content;
+          if (
+            retainedContent.some((block, index) => {
+              const projectedBlock = projectedContent[index];
+              return (
+                isText(block) && block.text && (!isText(projectedBlock) || !projectedBlock.text)
+              );
+            })
+          ) {
+            break;
+          }
+          const projected = replaceToolResultContent(msg, [...projectedContent, ...notice]);
+          const projectedChars = estimateMessageCharsCached(projected, cache);
+          if (projectedChars <= maxChars) {
+            return projected;
+          }
+          const adjustedTextBudget = Math.floor(
+            (textBudget * availableTextChars) / (projectedChars - reservedChars),
+          );
+          if (adjustedTextBudget <= 0 || adjustedTextBudget >= textBudget) {
+            break;
+          }
+          textBudget = adjustedTextBudget;
+        }
+      }
+      rawText = omissionNotice(0) + (rawText ? `\n\n${rawText}` : "");
+    }
+  }
 
-  const rawText = getToolResultText(msg);
   if (!rawText) {
     const omittedChars = Math.max(
       1,
       estimateBudgetToRawChars(Math.max(estimatedChars - maxChars, 1)),
     );
-    return replaceToolResultText(msg, formatContextLimitTruncationNotice(omittedChars));
+    return replaceToolResultContent(msg, formatContextLimitTruncationNotice(omittedChars));
   }
 
   if (maxChars <= 0) {
-    return replaceToolResultText(msg, formatContextLimitTruncationNotice(rawText.length));
+    return replaceToolResultContent(msg, formatContextLimitTruncationNotice(rawText.length));
   }
 
   const truncatedText = truncateToolResultText(rawText, maxChars, {
@@ -179,7 +257,7 @@ function truncateToolResultToChars(
     minimumRawWeight: TOOL_RESULT_CHARS_PER_TOKEN_ESTIMATE,
     preserveImportantTail: false,
   });
-  return replaceToolResultText(msg, truncatedText);
+  return replaceToolResultContent(msg, truncatedText);
 }
 
 function enforceToolResultLimit(params: {