fix(gateway): reject malformed active image inputs (#116883)

Co-authored-by: Peter Steinberger <steipete@macos.shared>

src/gateway/openai-http.test.ts

@@ -205,9 +205,9 @@ describe("OpenAI-compatible HTTP API (e2e)", () => {
       });
       expect(res.status).toBe(400);
       const json = (await res.json()) as Record<string, unknown>;
-      expect((json.error as Record<string, unknown> | undefined)?.type).toBe(
-        "invalid_request_error",
-      );
+      const error = json.error as Record<string, unknown> | undefined;
+      expect(error?.type).toBe("invalid_request_error");
+      expect(error?.message).toBe("Invalid image_url content in `messages`.");
       expect(agentCommand).toHaveBeenCalledTimes(0);
     };
     const postSyncUserMessage = async (message: string) => {
@@ -561,6 +561,48 @@ describe("OpenAI-compatible HTTP API (e2e)", () => {
         },
       ]);
 
+      const malformedImageParts = [
+        { type: "image_url" },
+        { type: "image_url", image_url: null },
+        { type: "image_url", image_url: {} },
+        { type: "image_url", image_url: { url: "   " } },
+        { type: "image_url", image_url: { url: 123 } },
+        { type: "image_url", image_url: { url: null } },
+        { type: "image_url", image_url: "   " },
+        { type: "image_url", image_url: 123 },
+      ];
+      const validImagePart = {
+        type: "image_url",
+        image_url: { url: "data:image/png;base64,QUJDRA==" },
+      };
+      for (const imagePart of malformedImageParts) {
+        for (const content of [
+          [imagePart],
+          [{ type: "text", text: "describe this" }, imagePart],
+          [validImagePart, imagePart],
+        ]) {
+          await expectInvalidRequestNoDispatch([{ role: "user", content }]);
+        }
+      }
+
+      for (const malformedDataUri of [
+        "data:image/png,QUJDRA==",
+        "data:image/png;base64,",
+        "data:image/png;base64,%%%",
+        "data:image/svg+xml;base64,PHN2Zz4=",
+        "data:image/png;base64,JVBERi0xLjQK",
+      ]) {
+        await expectInvalidRequestNoDispatch([
+          {
+            role: "user",
+            content: [
+              { type: "text", text: "describe this" },
+              { type: "image_url", image_url: { url: malformedDataUri } },
+            ],
+          },
+        ]);
+      }
+
       {
         mockAgentOnce([{ text: "I can see the image" }]);
         const res = await postChatCompletions(port, {
@@ -610,6 +652,33 @@ describe("OpenAI-compatible HTTP API (e2e)", () => {
         await res.text();
       }
 
+      for (const historicalImageParts of [
+        [{ type: "image_url", image_url: { url: "   " } }],
+        [validImagePart, { type: "image_url", image_url: { url: "   " } }],
+      ]) {
+        for (const followup of [
+          { role: "user", content: "What color was it?" },
+          { role: "tool", content: "Vision tool says it is blue." },
+        ]) {
+          mockAgentOnce([{ text: "follow up answer" }]);
+          const res = await postChatCompletions(port, {
+            model: "openclaw",
+            messages: [
+              {
+                role: "user",
+                content: [{ type: "text", text: "look at this" }, ...historicalImageParts],
+              },
+              { role: "assistant", content: "Checking the image." },
+              followup,
+            ],
+          });
+          expect(res.status).toBe(200);
+          expect(getFirstAgentCall()?.images).toBeUndefined();
+          expect(getFirstAgentMessage()).toContain("User: look at this");
+          await res.text();
+        }
+      }
+
       {
         mockAgentOnce([{ text: "latest image only" }]);
         const res = await postChatCompletions(port, {

src/gateway/openai-http.ts

@@ -520,11 +520,13 @@ function resolveImageUrlPart(part: unknown): string | undefined {
   return trimmed.length > 0 ? trimmed : undefined;
 }
 
-function extractImageUrls(content: unknown): string[] {
-  if (!Array.isArray(content)) {
-    return [];
-  }
+type ExtractedImageUrls = { kind: "valid"; urls: string[] } | { kind: "invalid" };
+
+function extractImageUrls(content: unknown): ExtractedImageUrls {
   const urls: string[] = [];
+  if (!Array.isArray(content)) {
+    return { kind: "valid", urls };
+  }
   for (const part of content) {
     if (!part || typeof part !== "object") {
       continue;
@@ -533,17 +535,18 @@ function extractImageUrls(content: unknown): string[] {
       continue;
     }
     const url = resolveImageUrlPart(part);
-    if (url) {
-      urls.push(url);
+    if (!url) {
+      return { kind: "invalid" };
     }
+    urls.push(url);
   }
-  return urls;
+  return { kind: "valid", urls };
 }
 
 type ActiveTurnContext = {
   activeTurnIndex: number;
   activeUserMessageIndex: number;
-  urls: string[];
+  imageUrls: ExtractedImageUrls;
 };
 
 function parseImageUrlToSource(url: string): InputImageSource {
@@ -586,20 +589,29 @@ function resolveActiveTurnContext(messagesUnknown: unknown): ActiveTurnContext {
     if (normalizedRole !== "user" && normalizedRole !== "tool") {
       continue;
     }
+    const imageUrls: ExtractedImageUrls =
+      normalizedRole === "user" ? extractImageUrls(msg.content) : { kind: "valid", urls: [] };
     return {
       activeTurnIndex: i,
       activeUserMessageIndex: normalizedRole === "user" ? i : -1,
-      urls: normalizedRole === "user" ? extractImageUrls(msg.content) : [],
+      imageUrls,
     };
   }
-  return { activeTurnIndex: -1, activeUserMessageIndex: -1, urls: [] };
+  return {
+    activeTurnIndex: -1,
+    activeUserMessageIndex: -1,
+    imageUrls: { kind: "valid", urls: [] },
+  };
 }
 
 async function resolveImagesForRequest(
-  activeTurnContext: Pick<ActiveTurnContext, "urls">,
+  activeTurnContext: Pick<ActiveTurnContext, "imageUrls">,
   limits: ResolvedOpenAiChatCompletionsLimits,
 ): Promise<ImageContent[]> {
-  const urls = activeTurnContext.urls;
+  if (activeTurnContext.imageUrls.kind === "invalid") {
+    throw new Error("image_url part is missing a valid URL");
+  }
+  const urls = activeTurnContext.imageUrls.urls;
   if (urls.length === 0) {
     return [];
   }
@@ -640,12 +652,14 @@ export const testOnlyOpenAiHttp = {
 
 function buildAgentPrompt(
   messagesUnknown: unknown,
-  activeUserMessageIndex: number,
+  activeTurnContext: Pick<ActiveTurnContext, "activeUserMessageIndex" | "imageUrls">,
 ): {
   message: string;
   extraSystemPrompt?: string;
 } {
   const messages = asMessages(messagesUnknown);
+  const hasActiveTurnImage =
+    activeTurnContext.imageUrls.kind === "valid" && activeTurnContext.imageUrls.urls.length > 0;
 
   const systemParts: string[] = [];
   const conversationEntries: ConversationEntry[] = [];
@@ -656,7 +670,6 @@ function buildAgentPrompt(
     }
     const role = normalizeOptionalString(msg.role) ?? "";
     const content = extractTextContent(msg.content).trim();
-    const hasImage = extractImageUrls(msg.content).length > 0;
     if (!role) {
       continue;
     }
@@ -679,7 +692,10 @@ function buildAgentPrompt(
     // Keep the image-only placeholder scoped to the active user turn so we don't
     // mention historical image-only turns whose bytes are intentionally not replayed.
     const baseMessageContent =
-      normalizedRole === "user" && !content && hasImage && i === activeUserMessageIndex
+      normalizedRole === "user" &&
+      !content &&
+      hasActiveTurnImage &&
+      i === activeTurnContext.activeUserMessageIndex
         ? IMAGE_ONLY_USER_MESSAGE
         : content;
     const messageContent = [baseMessageContent, assistantToolCallsSummary]
@@ -1037,7 +1053,7 @@ export async function handleOpenAiHttpRequest(
     return true;
   }
   const activeTurnContext = resolveActiveTurnContext(payload.messages);
-  const prompt = buildAgentPrompt(payload.messages, activeTurnContext.activeUserMessageIndex);
+  const prompt = buildAgentPrompt(payload.messages, activeTurnContext);
   let resolvedClientTools: ClientToolDefinition[];
   let toolChoicePrompt: string | undefined;
   let toolChoiceConstraint: ToolChoiceConstraint | undefined;