mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 13:26:04 -06:00
fix(litellm): send image edits as multipart/form-data (#118562)
The litellm image provider built edit requests as a JSON body with
`images: [{image_url}]`, but LiteLLM's POST /v1/images/edits is a multipart
endpoint following OpenAI's edits schema: the reference image must be an
uploaded file part named `image` (or `image[]`). LiteLLM never finds an `image`
key in a JSON body and fails before contacting the upstream provider:
HTTP 500 aimage_edit() missing 1 required positional argument: 'image'
so every image edit through this provider fails. Plain generation is unaffected.
No JSON variant works. Verified against LiteLLM v1.82.3 with a real 512x512 PNG:
plural `images` 500s as above; singular `image` as a data URL or bare base64,
string or array, returns HTTP 400 "Invalid image file or mode". Only multipart
succeeds. Note that singular `image` clears the 500 and reaches the provider,
which looks like progress but never delivers usable bytes.
Switch buildEditRequest to return { kind: "multipart", form }, which
createOpenAiCompatibleImageGenerationProvider already routes to
postMultipartRequest — matching the built-in openai and deepinfra providers.
A single reference uses the `image` part name and multiple use repeated
`image[]` parts; LiteLLM accepts either as List[UploadFile] and merges them,
erroring only when both appear in one request.
Also drops the now-unused imageToDataUrl helper and its imports.
Tests: the existing edit test asserted the buggy JSON shape, so it now asserts
a multipart request, that postJsonRequest is not called, and the single-image
part name; a new test covers repeated image[] parts for multiple references.
Verified end-to-end against LiteLLM v1.82.3 with a live agent: single-reference
and multi-image edits both return real images; both failed with the 500 before.
This commit is contained in:
@@ -31,6 +31,15 @@ function mockGeneratedPngResponse() {
|
||||
});
|
||||
}
|
||||
|
||||
function mockEditedPngResponse() {
|
||||
postMultipartRequestMock.mockResolvedValue({
|
||||
response: jsonResponse({
|
||||
data: [{ b64_json: Buffer.from("png-bytes").toString("base64") }],
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
}
|
||||
|
||||
function mockObjectArg(mock: unknown, index = -1): Record<string, unknown> {
|
||||
const calls = (mock as { mock?: { calls?: Array<Array<unknown>> } }).mock?.calls ?? [];
|
||||
const call = index < 0 ? calls.at(index) : calls[index];
|
||||
@@ -147,8 +156,8 @@ describe("litellm image generation provider", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("routes to the edit endpoint when input images are provided", async () => {
|
||||
mockGeneratedPngResponse();
|
||||
it("routes to the edit endpoint as multipart when input images are provided", async () => {
|
||||
mockEditedPngResponse();
|
||||
|
||||
const provider = buildLitellmImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
@@ -164,9 +173,40 @@ describe("litellm image generation provider", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(mockObjectArg(postJsonRequestMock).url).toBe("http://localhost:4000/images/edits");
|
||||
const call = postJsonRequestMock.mock.calls[0]?.[0] as { body: { images: unknown[] } };
|
||||
expect(call.body.images).toHaveLength(1);
|
||||
// Edits must be multipart, never JSON: LiteLLM's /images/edits maps onto
|
||||
// `aimage_edit(image=...)` and rejects a JSON body outright.
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
expect(mockObjectArg(postMultipartRequestMock).url).toBe("http://localhost:4000/images/edits");
|
||||
|
||||
const form = mockObjectArg(postMultipartRequestMock).body as FormData;
|
||||
expect(form.get("model")).toBe("gpt-image-2");
|
||||
expect(form.get("prompt")).toBe("refine the hero");
|
||||
// A single reference uses the singular `image` part name.
|
||||
expect(form.getAll("image")).toHaveLength(1);
|
||||
expect(form.getAll("image[]")).toHaveLength(0);
|
||||
expect(form.get("image")).toBeInstanceOf(Blob);
|
||||
});
|
||||
|
||||
it("sends multiple reference images as repeated image[] parts", async () => {
|
||||
mockEditedPngResponse();
|
||||
|
||||
const provider = buildLitellmImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "litellm",
|
||||
model: "gpt-image-2",
|
||||
prompt: "merge these",
|
||||
cfg: {},
|
||||
inputImages: [
|
||||
{ buffer: Buffer.from("first"), mimeType: "image/png" },
|
||||
{ buffer: Buffer.from("second"), mimeType: "image/jpeg" },
|
||||
],
|
||||
});
|
||||
|
||||
const form = mockObjectArg(postMultipartRequestMock).body as FormData;
|
||||
// Both names are accepted by OpenAI-compatible edit endpoints, but only one
|
||||
// may be present per request — sending both is an error.
|
||||
expect(form.getAll("image[]")).toHaveLength(2);
|
||||
expect(form.getAll("image")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("throws a clear error when the API key is missing", async () => {
|
||||
|
||||
@@ -3,9 +3,8 @@ import { isIP } from "node:net";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import {
|
||||
createOpenAiCompatibleImageGenerationProvider,
|
||||
imageSourceUploadFileName,
|
||||
type ImageGenerationProvider,
|
||||
type ImageGenerationSourceImage,
|
||||
toImageDataUrl,
|
||||
} from "openclaw/plugin-sdk/image-generation";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { LITELLM_BASE_URL } from "./onboard.js";
|
||||
@@ -41,10 +40,6 @@ function resolveConfiguredLitellmBaseUrl(cfg: OpenClawConfig | undefined): strin
|
||||
return normalizeOptionalString(resolveLitellmProviderConfig(cfg)?.baseUrl) ?? LITELLM_BASE_URL;
|
||||
}
|
||||
|
||||
function imageToDataUrl(image: ImageGenerationSourceImage): string {
|
||||
return toImageDataUrl({ buffer: image.buffer, mimeType: image.mimeType });
|
||||
}
|
||||
|
||||
// LiteLLM's default proxy is loopback. Auto-enable private-network access only
|
||||
// for loopback-style hosts; LAN/custom private endpoints should use the
|
||||
// explicit models.providers.litellm.request.allowPrivateNetwork opt-in.
|
||||
@@ -124,18 +119,28 @@ export function buildLitellmImageGenerationProvider(): ImageGenerationProvider {
|
||||
size: req.size ?? DEFAULT_SIZE,
|
||||
},
|
||||
}),
|
||||
buildEditRequest: ({ req, inputImages, model, count }) => ({
|
||||
kind: "json",
|
||||
body: {
|
||||
model,
|
||||
prompt: req.prompt,
|
||||
n: count,
|
||||
size: req.size ?? DEFAULT_SIZE,
|
||||
images: inputImages.map((image) => ({
|
||||
image_url: imageToDataUrl(image),
|
||||
})),
|
||||
},
|
||||
}),
|
||||
// LiteLLM's /v1/images/edits is multipart (OpenAI's edits schema): the
|
||||
// reference image must be an uploaded file part, not a JSON field — a JSON
|
||||
// body fails before the request reaches the provider.
|
||||
buildEditRequest: ({ req, inputImages, model, count }) => {
|
||||
const form = new FormData();
|
||||
form.set("model", model);
|
||||
form.set("prompt", req.prompt);
|
||||
form.set("n", String(count));
|
||||
form.set("size", req.size ?? DEFAULT_SIZE);
|
||||
// OpenAI-compatible edits take repeated `image[]` parts when more than one
|
||||
// reference is supplied, and a single `image` part otherwise.
|
||||
const partName = inputImages.length > 1 ? "image[]" : "image";
|
||||
for (const [index, image] of inputImages.entries()) {
|
||||
const mimeType = normalizeOptionalString(image.mimeType) ?? "image/png";
|
||||
form.append(
|
||||
partName,
|
||||
new Blob([new Uint8Array(image.buffer)], { type: mimeType }),
|
||||
imageSourceUploadFileName({ image, index }),
|
||||
);
|
||||
}
|
||||
return { kind: "multipart", form };
|
||||
},
|
||||
missingApiKeyError: "LiteLLM API key missing",
|
||||
failureLabels: {
|
||||
generate: "LiteLLM image generation failed",
|
||||
|
||||
Reference in New Issue
Block a user