fix(litellm): send image edits as multipart/form-data (#118562)

The litellm image provider built edit requests as a JSON body with
`images: [{image_url}]`, but LiteLLM's POST /v1/images/edits is a multipart
endpoint following OpenAI's edits schema: the reference image must be an
uploaded file part named `image` (or `image[]`). LiteLLM never finds an `image`
key in a JSON body and fails before contacting the upstream provider:

  HTTP 500 aimage_edit() missing 1 required positional argument: 'image'

so every image edit through this provider fails. Plain generation is unaffected.

No JSON variant works. Verified against LiteLLM v1.82.3 with a real 512x512 PNG:
plural `images` 500s as above; singular `image` as a data URL or bare base64,
string or array, returns HTTP 400 "Invalid image file or mode". Only multipart
succeeds. Note that singular `image` clears the 500 and reaches the provider,
which looks like progress but never delivers usable bytes.

Switch buildEditRequest to return { kind: "multipart", form }, which
createOpenAiCompatibleImageGenerationProvider already routes to
postMultipartRequest — matching the built-in openai and deepinfra providers.
A single reference uses the `image` part name and multiple use repeated
`image[]` parts; LiteLLM accepts either as List[UploadFile] and merges them,
erroring only when both appear in one request.

Also drops the now-unused imageToDataUrl helper and its imports.

Tests: the existing edit test asserted the buggy JSON shape, so it now asserts
a multipart request, that postJsonRequest is not called, and the single-image
part name; a new test covers repeated image[] parts for multiple references.

Verified end-to-end against LiteLLM v1.82.3 with a live agent: single-reference
and multi-image edits both return real images; both failed with the 500 before.
This commit is contained in:
Sukhdeep
2026-08-03 17:59:08 +08:00
committed by GitHub
parent deb682abfe
commit a7396255d0
2 changed files with 68 additions and 23 deletions
@@ -31,6 +31,15 @@ function mockGeneratedPngResponse() {
});
}
function mockEditedPngResponse() {
postMultipartRequestMock.mockResolvedValue({
response: jsonResponse({
data: [{ b64_json: Buffer.from("png-bytes").toString("base64") }],
}),
release: vi.fn(async () => {}),
});
}
function mockObjectArg(mock: unknown, index = -1): Record<string, unknown> {
const calls = (mock as { mock?: { calls?: Array<Array<unknown>> } }).mock?.calls ?? [];
const call = index < 0 ? calls.at(index) : calls[index];
@@ -147,8 +156,8 @@ describe("litellm image generation provider", () => {
});
});
it("routes to the edit endpoint when input images are provided", async () => {
mockGeneratedPngResponse();
it("routes to the edit endpoint as multipart when input images are provided", async () => {
mockEditedPngResponse();
const provider = buildLitellmImageGenerationProvider();
await provider.generateImage({
@@ -164,9 +173,40 @@ describe("litellm image generation provider", () => {
],
});
expect(mockObjectArg(postJsonRequestMock).url).toBe("http://localhost:4000/images/edits");
const call = postJsonRequestMock.mock.calls[0]?.[0] as { body: { images: unknown[] } };
expect(call.body.images).toHaveLength(1);
// Edits must be multipart, never JSON: LiteLLM's /images/edits maps onto
// `aimage_edit(image=...)` and rejects a JSON body outright.
expect(postJsonRequestMock).not.toHaveBeenCalled();
expect(mockObjectArg(postMultipartRequestMock).url).toBe("http://localhost:4000/images/edits");
const form = mockObjectArg(postMultipartRequestMock).body as FormData;
expect(form.get("model")).toBe("gpt-image-2");
expect(form.get("prompt")).toBe("refine the hero");
// A single reference uses the singular `image` part name.
expect(form.getAll("image")).toHaveLength(1);
expect(form.getAll("image[]")).toHaveLength(0);
expect(form.get("image")).toBeInstanceOf(Blob);
});
it("sends multiple reference images as repeated image[] parts", async () => {
mockEditedPngResponse();
const provider = buildLitellmImageGenerationProvider();
await provider.generateImage({
provider: "litellm",
model: "gpt-image-2",
prompt: "merge these",
cfg: {},
inputImages: [
{ buffer: Buffer.from("first"), mimeType: "image/png" },
{ buffer: Buffer.from("second"), mimeType: "image/jpeg" },
],
});
const form = mockObjectArg(postMultipartRequestMock).body as FormData;
// Both names are accepted by OpenAI-compatible edit endpoints, but only one
// may be present per request — sending both is an error.
expect(form.getAll("image[]")).toHaveLength(2);
expect(form.getAll("image")).toHaveLength(0);
});
it("throws a clear error when the API key is missing", async () => {
+23 -18
View File
@@ -3,9 +3,8 @@ import { isIP } from "node:net";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import {
createOpenAiCompatibleImageGenerationProvider,
imageSourceUploadFileName,
type ImageGenerationProvider,
type ImageGenerationSourceImage,
toImageDataUrl,
} from "openclaw/plugin-sdk/image-generation";
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
import { LITELLM_BASE_URL } from "./onboard.js";
@@ -41,10 +40,6 @@ function resolveConfiguredLitellmBaseUrl(cfg: OpenClawConfig | undefined): strin
return normalizeOptionalString(resolveLitellmProviderConfig(cfg)?.baseUrl) ?? LITELLM_BASE_URL;
}
function imageToDataUrl(image: ImageGenerationSourceImage): string {
return toImageDataUrl({ buffer: image.buffer, mimeType: image.mimeType });
}
// LiteLLM's default proxy is loopback. Auto-enable private-network access only
// for loopback-style hosts; LAN/custom private endpoints should use the
// explicit models.providers.litellm.request.allowPrivateNetwork opt-in.
@@ -124,18 +119,28 @@ export function buildLitellmImageGenerationProvider(): ImageGenerationProvider {
size: req.size ?? DEFAULT_SIZE,
},
}),
buildEditRequest: ({ req, inputImages, model, count }) => ({
kind: "json",
body: {
model,
prompt: req.prompt,
n: count,
size: req.size ?? DEFAULT_SIZE,
images: inputImages.map((image) => ({
image_url: imageToDataUrl(image),
})),
},
}),
// LiteLLM's /v1/images/edits is multipart (OpenAI's edits schema): the
// reference image must be an uploaded file part, not a JSON field — a JSON
// body fails before the request reaches the provider.
buildEditRequest: ({ req, inputImages, model, count }) => {
const form = new FormData();
form.set("model", model);
form.set("prompt", req.prompt);
form.set("n", String(count));
form.set("size", req.size ?? DEFAULT_SIZE);
// OpenAI-compatible edits take repeated `image[]` parts when more than one
// reference is supplied, and a single `image` part otherwise.
const partName = inputImages.length > 1 ? "image[]" : "image";
for (const [index, image] of inputImages.entries()) {
const mimeType = normalizeOptionalString(image.mimeType) ?? "image/png";
form.append(
partName,
new Blob([new Uint8Array(image.buffer)], { type: mimeType }),
imageSourceUploadFileName({ image, index }),
);
}
return { kind: "multipart", form };
},
missingApiKeyError: "LiteLLM API key missing",
failureLabels: {
generate: "LiteLLM image generation failed",