diff --git a/src/gateway/openai-http.test.ts b/src/gateway/openai-http.test.ts index 0c3bf6671ccb..7f6a3f089ad5 100644 --- a/src/gateway/openai-http.test.ts +++ b/src/gateway/openai-http.test.ts @@ -205,9 +205,9 @@ describe("OpenAI-compatible HTTP API (e2e)", () => { }); expect(res.status).toBe(400); const json = (await res.json()) as Record; - expect((json.error as Record | undefined)?.type).toBe( - "invalid_request_error", - ); + const error = json.error as Record | undefined; + expect(error?.type).toBe("invalid_request_error"); + expect(error?.message).toBe("Invalid image_url content in `messages`."); expect(agentCommand).toHaveBeenCalledTimes(0); }; const postSyncUserMessage = async (message: string) => { @@ -561,6 +561,48 @@ describe("OpenAI-compatible HTTP API (e2e)", () => { }, ]); + const malformedImageParts = [ + { type: "image_url" }, + { type: "image_url", image_url: null }, + { type: "image_url", image_url: {} }, + { type: "image_url", image_url: { url: " " } }, + { type: "image_url", image_url: { url: 123 } }, + { type: "image_url", image_url: { url: null } }, + { type: "image_url", image_url: " " }, + { type: "image_url", image_url: 123 }, + ]; + const validImagePart = { + type: "image_url", + image_url: { url: "data:image/png;base64,QUJDRA==" }, + }; + for (const imagePart of malformedImageParts) { + for (const content of [ + [imagePart], + [{ type: "text", text: "describe this" }, imagePart], + [validImagePart, imagePart], + ]) { + await expectInvalidRequestNoDispatch([{ role: "user", content }]); + } + } + + for (const malformedDataUri of [ + "data:image/png,QUJDRA==", + "data:image/png;base64,", + "data:image/png;base64,%%%", + "data:image/svg+xml;base64,PHN2Zz4=", + "data:image/png;base64,JVBERi0xLjQK", + ]) { + await expectInvalidRequestNoDispatch([ + { + role: "user", + content: [ + { type: "text", text: "describe this" }, + { type: "image_url", image_url: { url: malformedDataUri } }, + ], + }, + ]); + } + { mockAgentOnce([{ text: "I can see the image" }]); const res = await postChatCompletions(port, { @@ -610,6 +652,33 @@ describe("OpenAI-compatible HTTP API (e2e)", () => { await res.text(); } + for (const historicalImageParts of [ + [{ type: "image_url", image_url: { url: " " } }], + [validImagePart, { type: "image_url", image_url: { url: " " } }], + ]) { + for (const followup of [ + { role: "user", content: "What color was it?" }, + { role: "tool", content: "Vision tool says it is blue." }, + ]) { + mockAgentOnce([{ text: "follow up answer" }]); + const res = await postChatCompletions(port, { + model: "openclaw", + messages: [ + { + role: "user", + content: [{ type: "text", text: "look at this" }, ...historicalImageParts], + }, + { role: "assistant", content: "Checking the image." }, + followup, + ], + }); + expect(res.status).toBe(200); + expect(getFirstAgentCall()?.images).toBeUndefined(); + expect(getFirstAgentMessage()).toContain("User: look at this"); + await res.text(); + } + } + { mockAgentOnce([{ text: "latest image only" }]); const res = await postChatCompletions(port, { diff --git a/src/gateway/openai-http.ts b/src/gateway/openai-http.ts index a82d660ab7c2..47e9c046acf5 100644 --- a/src/gateway/openai-http.ts +++ b/src/gateway/openai-http.ts @@ -520,11 +520,13 @@ function resolveImageUrlPart(part: unknown): string | undefined { return trimmed.length > 0 ? trimmed : undefined; } -function extractImageUrls(content: unknown): string[] { - if (!Array.isArray(content)) { - return []; - } +type ExtractedImageUrls = { kind: "valid"; urls: string[] } | { kind: "invalid" }; + +function extractImageUrls(content: unknown): ExtractedImageUrls { const urls: string[] = []; + if (!Array.isArray(content)) { + return { kind: "valid", urls }; + } for (const part of content) { if (!part || typeof part !== "object") { continue; @@ -533,17 +535,18 @@ function extractImageUrls(content: unknown): string[] { continue; } const url = resolveImageUrlPart(part); - if (url) { - urls.push(url); + if (!url) { + return { kind: "invalid" }; } + urls.push(url); } - return urls; + return { kind: "valid", urls }; } type ActiveTurnContext = { activeTurnIndex: number; activeUserMessageIndex: number; - urls: string[]; + imageUrls: ExtractedImageUrls; }; function parseImageUrlToSource(url: string): InputImageSource { @@ -586,20 +589,29 @@ function resolveActiveTurnContext(messagesUnknown: unknown): ActiveTurnContext { if (normalizedRole !== "user" && normalizedRole !== "tool") { continue; } + const imageUrls: ExtractedImageUrls = + normalizedRole === "user" ? extractImageUrls(msg.content) : { kind: "valid", urls: [] }; return { activeTurnIndex: i, activeUserMessageIndex: normalizedRole === "user" ? i : -1, - urls: normalizedRole === "user" ? extractImageUrls(msg.content) : [], + imageUrls, }; } - return { activeTurnIndex: -1, activeUserMessageIndex: -1, urls: [] }; + return { + activeTurnIndex: -1, + activeUserMessageIndex: -1, + imageUrls: { kind: "valid", urls: [] }, + }; } async function resolveImagesForRequest( - activeTurnContext: Pick, + activeTurnContext: Pick, limits: ResolvedOpenAiChatCompletionsLimits, ): Promise { - const urls = activeTurnContext.urls; + if (activeTurnContext.imageUrls.kind === "invalid") { + throw new Error("image_url part is missing a valid URL"); + } + const urls = activeTurnContext.imageUrls.urls; if (urls.length === 0) { return []; } @@ -640,12 +652,14 @@ export const testOnlyOpenAiHttp = { function buildAgentPrompt( messagesUnknown: unknown, - activeUserMessageIndex: number, + activeTurnContext: Pick, ): { message: string; extraSystemPrompt?: string; } { const messages = asMessages(messagesUnknown); + const hasActiveTurnImage = + activeTurnContext.imageUrls.kind === "valid" && activeTurnContext.imageUrls.urls.length > 0; const systemParts: string[] = []; const conversationEntries: ConversationEntry[] = []; @@ -656,7 +670,6 @@ function buildAgentPrompt( } const role = normalizeOptionalString(msg.role) ?? ""; const content = extractTextContent(msg.content).trim(); - const hasImage = extractImageUrls(msg.content).length > 0; if (!role) { continue; } @@ -679,7 +692,10 @@ function buildAgentPrompt( // Keep the image-only placeholder scoped to the active user turn so we don't // mention historical image-only turns whose bytes are intentionally not replayed. const baseMessageContent = - normalizedRole === "user" && !content && hasImage && i === activeUserMessageIndex + normalizedRole === "user" && + !content && + hasActiveTurnImage && + i === activeTurnContext.activeUserMessageIndex ? IMAGE_ONLY_USER_MESSAGE : content; const messageContent = [baseMessageContent, assistantToolCallsSummary] @@ -1037,7 +1053,7 @@ export async function handleOpenAiHttpRequest( return true; } const activeTurnContext = resolveActiveTurnContext(payload.messages); - const prompt = buildAgentPrompt(payload.messages, activeTurnContext.activeUserMessageIndex); + const prompt = buildAgentPrompt(payload.messages, activeTurnContext); let resolvedClientTools: ClientToolDefinition[]; let toolChoicePrompt: string | undefined; let toolChoiceConstraint: ToolChoiceConstraint | undefined;