diff --git a/src/gateway/openresponses-http.server.file-only.test.ts b/src/gateway/openresponses-http.server.file-only.test.ts new file mode 100644 index 000000000000..3928eba1e168 --- /dev/null +++ b/src/gateway/openresponses-http.server.file-only.test.ts @@ -0,0 +1,136 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +const extractFileContentFromSourceMock = vi.fn(); + +vi.mock("../media/input-files.js", async () => { + const actual = + await vi.importActual("../media/input-files.js"); + return { + ...actual, + extractFileContentFromSource: (...args: unknown[]) => extractFileContentFromSourceMock(...args), + }; +}); + +import { + agentCommand, + getFreePort, + installGatewayTestHooks, + startGatewayServerWithRetries, +} from "./test-helpers.js"; + +installGatewayTestHooks({ scope: "suite" }); + +let server: Awaited>["server"]; +let port: number; + +beforeAll(async () => { + const started = await startGatewayServerWithRetries({ + port: await getFreePort(), + opts: { + host: "127.0.0.1", + auth: { mode: "none" }, + controlUiEnabled: false, + openResponsesEnabled: true, + }, + }); + port = started.port; + server = started.server; +}); + +afterAll(async () => { + await server?.close({ reason: "openresponses file-only suite done" }); +}); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +async function postResponses(body: unknown) { + return await fetch(`http://127.0.0.1:${port}/v1/responses`, { + method: "POST", + headers: { + "content-type": "application/json", + "x-openclaw-scopes": "operator.write", + }, + body: JSON.stringify(body), + }); +} + +describe("OpenResponses file-only input that renders to images", () => { + it("accepts a file-only turn whose file renders to images and forwards them", async () => { + extractFileContentFromSourceMock.mockResolvedValueOnce({ + filename: "scan.pdf", + text: "", + images: [ + { type: "image", data: Buffer.alloc(8, 1).toString("base64"), mimeType: "image/png" }, + ], + }); + agentCommand.mockResolvedValueOnce({ payloads: [{ text: "ok" }] } as never); + + const res = await postResponses({ + model: "openclaw", + instructions: "Describe the attached scan.", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_file", + source: { + type: "base64", + media_type: "application/pdf", + data: Buffer.from("%PDF-1.4 scanned").toString("base64"), + filename: "scan.pdf", + }, + }, + ], + }, + ], + }); + + expect(res.status).toBe(200); + expect(agentCommand).toHaveBeenCalledTimes(1); + const opts = agentCommand.mock.calls[0]?.[0] as { message?: string; images?: unknown[] }; + expect(opts.message ?? "").not.toBe(""); + expect(opts.images?.length).toBe(1); + await res.text(); + }); + + it("keeps an empty extracted file visible to the model", async () => { + extractFileContentFromSourceMock.mockResolvedValueOnce({ + filename: "empty.txt", + text: "", + images: [], + }); + agentCommand.mockResolvedValueOnce({ payloads: [{ text: "ok" }] } as never); + + const res = await postResponses({ + model: "openclaw", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_file", + source: { + type: "base64", + media_type: "text/plain", + data: Buffer.from("binary-only file").toString("base64"), + filename: "empty.txt", + }, + }, + ], + }, + ], + }); + + const body = await res.text(); + expect(res.status, body).toBe(200); + expect(agentCommand).toHaveBeenCalledTimes(1); + const opts = agentCommand.mock.calls[0]?.[0] as { extraSystemPrompt?: string }; + expect(opts.extraSystemPrompt).toContain(''); + expect(opts.extraSystemPrompt).toContain("[No extractable text]"); + }); +}); diff --git a/src/gateway/openresponses-http.test.ts b/src/gateway/openresponses-http.test.ts index c7c53b5aa144..6cba28ecc3b1 100644 --- a/src/gateway/openresponses-http.test.ts +++ b/src/gateway/openresponses-http.test.ts @@ -1746,6 +1746,43 @@ describe("OpenResponses HTTP API (e2e)", () => { await ensureResponseConsumed(res); }); + it("accepts file-only input without text, matching image-only", async () => { + const port = enabledPort; + agentCommand.mockClear(); + agentCommand.mockResolvedValueOnce({ payloads: [{ text: "ok" }] } as never); + + const res = await postResponses(port, { + model: "openclaw", + instructions: "Summarize the attached document.", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_file", + source: { + type: "base64", + media_type: "text/plain", + data: Buffer.from("the quick brown fox").toString("base64"), + filename: "doc.txt", + }, + }, + ], + }, + ], + }); + + expect(res.status).toBe(200); + expect(agentCommand).toHaveBeenCalledTimes(1); + const opts = firstAgentOpts(); + expect((opts as { message?: string }).message ?? "").not.toBe(""); + const extraSystemPrompt = (opts as { extraSystemPrompt?: string }).extraSystemPrompt ?? ""; + expect(extraSystemPrompt).toContain(''); + expect(extraSystemPrompt).toContain("the quick brown fox"); + await ensureResponseConsumed(res); + }); + it("still rejects input with neither text nor image", async () => { const port = enabledPort; agentCommand.mockClear(); diff --git a/src/gateway/openresponses-http.ts b/src/gateway/openresponses-http.ts index b9087ca302c0..69f6875cf7cc 100644 --- a/src/gateway/openresponses-http.ts +++ b/src/gateway/openresponses-http.ts @@ -612,6 +612,14 @@ export async function handleOpenResponsesHttpRequest( surroundContentWithNewlines: false, }), ); + } else { + fileContexts.push( + renderFileContextBlock({ + filename: file.filename, + content: "[No extractable text]", + surroundContentWithNewlines: false, + }), + ); } if (file.images && file.images.length > 0) { images = images.concat(file.images); diff --git a/src/gateway/openresponses-parity.test.ts b/src/gateway/openresponses-parity.test.ts index d2ab8a5c733d..780601e85c5e 100644 --- a/src/gateway/openresponses-parity.test.ts +++ b/src/gateway/openresponses-parity.test.ts @@ -393,7 +393,43 @@ describe("OpenResponses Feature Parity", () => { expect(result.message).toBe(IMAGE_ONLY_USER_MESSAGE); }); - it("keeps an empty message when the active turn has neither text nor image", () => { + it("substitutes a placeholder for a file-only active user turn", () => { + const result = buildAgentPrompt([ + { + type: "message" as const, + role: "user" as const, + content: [ + { + type: "input_file" as const, + source: { type: "url" as const, url: "https://example.com/report.pdf" }, + }, + ], + }, + ]); + + expect(result.message).not.toBe(""); + expect(result.message.toLowerCase()).toContain("file"); + }); + + it("keeps the user text when a file-only turn also carries text", () => { + const result = buildAgentPrompt([ + { + type: "message" as const, + role: "user" as const, + content: [ + { type: "input_text" as const, text: "summarize this" }, + { + type: "input_file" as const, + source: { type: "url" as const, url: "https://example.com/report.pdf" }, + }, + ], + }, + ]); + + expect(result.message).toBe("summarize this"); + }); + + it("keeps an empty message when the active turn has neither text, image, nor file", () => { const result = buildAgentPrompt([ { type: "message" as const, role: "user" as const, content: [] }, ]); diff --git a/src/gateway/openresponses-prompt.ts b/src/gateway/openresponses-prompt.ts index 2a558c4ff23d..a80580149b08 100644 --- a/src/gateway/openresponses-prompt.ts +++ b/src/gateway/openresponses-prompt.ts @@ -6,6 +6,8 @@ import { } from "./agent-prompt.js"; import type { ContentPart, ItemParam } from "./open-responses.schema.js"; +const FILE_ONLY_USER_MESSAGE = "User sent file(s) with no text."; + function extractTextContent(content: string | ContentPart[]): string { if (typeof content === "string") { return content; @@ -28,6 +30,20 @@ function hasImageContent(content: string | ContentPart[]): boolean { return typeof content !== "string" && content.some((part) => part.type === "input_image"); } +function hasFileContent(content: string | ContentPart[]): boolean { + return typeof content !== "string" && content.some((part) => part.type === "input_file"); +} + +function placeholderForActiveTurn(content: string | ContentPart[]): string { + if (hasImageContent(content)) { + return IMAGE_ONLY_USER_MESSAGE; + } + if (hasFileContent(content)) { + return FILE_ONLY_USER_MESSAGE; + } + return ""; +} + /** Index of the last user message item, or -1 when there is none. */ function findActiveUserMessageIndex(input: ItemParam[]): number { for (let i = input.length - 1; i >= 0; i -= 1) { @@ -55,14 +71,15 @@ export function buildAgentPrompt(input: string | ItemParam[]): { for (const [i, item] of input.entries()) { if (item.type === "message") { const content = extractTextContent(item.content).trim(); - // Substitute a placeholder for an image-only active user turn so the turn - // is not dropped and the downstream agent command (which requires non-empty - // message text) still runs with the attached image, matching /v1/chat/completions. - // Historical image-only turns stay skipped because their bytes are not replayed. + // Substitute a placeholder for an image-only or file-only active user turn + // so the turn is not dropped and the downstream agent command (which requires + // non-empty message text) still runs with the attached image or file context, + // matching /v1/chat/completions. Historical media-only turns stay skipped + // because their bytes are not replayed. const body = content || - (item.role === "user" && i === activeUserMessageIndex && hasImageContent(item.content) - ? IMAGE_ONLY_USER_MESSAGE + (item.role === "user" && i === activeUserMessageIndex + ? placeholderForActiveTurn(item.content) : ""); if (!body) { continue;