mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
feat(microsoft-foundry): add MAI image provider
This commit is contained in:
Vincent Koc
@@ -0,0 +1,321 @@
|
||||
// Microsoft Foundry image provider tests cover MAI request construction.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildMicrosoftFoundryImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import { PROVIDER_ID } from "./shared.js";
|
||||
|
||||
const {
|
||||
assertOkOrThrowHttpErrorMock,
|
||||
isProviderApiKeyConfiguredMock,
|
||||
postJsonRequestMock,
|
||||
postMultipartRequestMock,
|
||||
prepareFoundryRuntimeAuthMock,
|
||||
resolveApiKeyForProviderMock,
|
||||
resolveProviderHttpRequestConfigMock,
|
||||
resolveProviderOperationTimeoutMsMock,
|
||||
sanitizeConfiguredModelProviderRequestMock,
|
||||
} = vi.hoisted(() => ({
|
||||
assertOkOrThrowHttpErrorMock: vi.fn(async () => {}),
|
||||
isProviderApiKeyConfiguredMock: vi.fn(() => true),
|
||||
postJsonRequestMock: vi.fn(),
|
||||
postMultipartRequestMock: vi.fn(),
|
||||
prepareFoundryRuntimeAuthMock: vi.fn(),
|
||||
resolveApiKeyForProviderMock: vi.fn(async () => ({
|
||||
apiKey: "foundry-key",
|
||||
mode: "api-key" as const,
|
||||
source: "test",
|
||||
})),
|
||||
resolveProviderHttpRequestConfigMock: vi.fn((params: Record<string, unknown>) => ({
|
||||
baseUrl: params.baseUrl ?? params.defaultBaseUrl,
|
||||
allowPrivateNetwork: false,
|
||||
headers: new Headers(params.defaultHeaders as HeadersInit | undefined),
|
||||
dispatcherPolicy: undefined,
|
||||
})),
|
||||
resolveProviderOperationTimeoutMsMock: vi.fn(
|
||||
(params: Record<string, unknown>) => params.timeoutMs ?? params.defaultTimeoutMs,
|
||||
),
|
||||
sanitizeConfiguredModelProviderRequestMock: vi.fn((request) => request),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-auth", () => ({
|
||||
isProviderApiKeyConfigured: isProviderApiKeyConfiguredMock,
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({
|
||||
resolveApiKeyForProvider: resolveApiKeyForProviderMock,
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/provider-http", () => ({
|
||||
assertOkOrThrowHttpError: assertOkOrThrowHttpErrorMock,
|
||||
postJsonRequest: postJsonRequestMock,
|
||||
postMultipartRequest: postMultipartRequestMock,
|
||||
resolveProviderHttpRequestConfig: resolveProviderHttpRequestConfigMock,
|
||||
resolveProviderOperationTimeoutMs: resolveProviderOperationTimeoutMsMock,
|
||||
sanitizeConfiguredModelProviderRequest: sanitizeConfiguredModelProviderRequestMock,
|
||||
}));
|
||||
|
||||
vi.mock("./runtime.js", () => ({
|
||||
prepareFoundryRuntimeAuth: prepareFoundryRuntimeAuthMock,
|
||||
}));
|
||||
|
||||
function buildConfig(
|
||||
params: {
|
||||
modelId?: string;
|
||||
modelName?: string;
|
||||
baseUrl?: string;
|
||||
} = {},
|
||||
): OpenClawConfig {
|
||||
const baseUrl = params.baseUrl ?? "https://example.services.ai.azure.com/openai/v1";
|
||||
const modelId = params.modelId ?? "image-deployment";
|
||||
const modelName = params.modelName ?? "MAI-Image-2.5";
|
||||
return {
|
||||
models: {
|
||||
providers: {
|
||||
[PROVIDER_ID]: {
|
||||
baseUrl,
|
||||
api: "openai-completions",
|
||||
models: [
|
||||
{
|
||||
provider: PROVIDER_ID,
|
||||
id: modelId,
|
||||
name: modelName,
|
||||
api: "openai-completions",
|
||||
baseUrl,
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 32_000,
|
||||
maxTokens: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function releasedJson(payload: unknown) {
|
||||
return {
|
||||
response: Response.json(payload),
|
||||
release: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
function requirePostJsonRequest(): Record<string, unknown> {
|
||||
const request = postJsonRequestMock.mock.calls[0]?.[0];
|
||||
if (!request || typeof request !== "object") {
|
||||
throw new Error("expected Microsoft Foundry JSON image request");
|
||||
}
|
||||
return request as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function requirePostMultipartRequest(): Record<string, unknown> {
|
||||
const request = postMultipartRequestMock.mock.calls[0]?.[0];
|
||||
if (!request || typeof request !== "object") {
|
||||
throw new Error("expected Microsoft Foundry multipart image request");
|
||||
}
|
||||
return request as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function requireHeaders(value: unknown): Headers {
|
||||
expect(value).toBeInstanceOf(Headers);
|
||||
if (!(value instanceof Headers)) {
|
||||
throw new Error("expected request headers");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
describe("microsoft foundry image generation provider", () => {
|
||||
afterEach(() => {
|
||||
assertOkOrThrowHttpErrorMock.mockClear();
|
||||
isProviderApiKeyConfiguredMock.mockClear();
|
||||
postJsonRequestMock.mockReset();
|
||||
postMultipartRequestMock.mockReset();
|
||||
prepareFoundryRuntimeAuthMock.mockReset();
|
||||
resolveApiKeyForProviderMock.mockClear();
|
||||
resolveProviderHttpRequestConfigMock.mockClear();
|
||||
resolveProviderOperationTimeoutMsMock.mockClear();
|
||||
sanitizeConfiguredModelProviderRequestMock.mockClear();
|
||||
});
|
||||
|
||||
it("exposes MAI image provider metadata and capabilities", () => {
|
||||
const provider = buildMicrosoftFoundryImageGenerationProvider();
|
||||
expect(provider.id).toBe(PROVIDER_ID);
|
||||
expect(provider.defaultModel).toBe("MAI-Image-2.5");
|
||||
expect(provider.models).toEqual([
|
||||
"MAI-Image-2.5-Flash",
|
||||
"MAI-Image-2.5",
|
||||
"MAI-Image-2e",
|
||||
"MAI-Image-2",
|
||||
]);
|
||||
expect(provider.capabilities.generate.maxCount).toBe(1);
|
||||
expect(provider.capabilities.edit.enabled).toBe(true);
|
||||
expect(provider.capabilities.edit.maxInputImages).toBe(1);
|
||||
expect(provider.capabilities.output?.formats).toEqual(["png"]);
|
||||
expect(provider.isConfigured?.({ agentDir: "/agent" })).toBe(true);
|
||||
expect(isProviderApiKeyConfiguredMock).toHaveBeenCalledWith({
|
||||
provider: PROVIDER_ID,
|
||||
agentDir: "/agent",
|
||||
});
|
||||
});
|
||||
|
||||
it("sends MAI image generation requests to the Foundry MAI endpoint with API-key auth", async () => {
|
||||
postJsonRequestMock.mockResolvedValue(
|
||||
releasedJson({
|
||||
data: [{ b64_json: Buffer.from("png").toString("base64") }],
|
||||
}),
|
||||
);
|
||||
const provider = buildMicrosoftFoundryImageGenerationProvider();
|
||||
|
||||
const result = await provider.generateImage({
|
||||
provider: PROVIDER_ID,
|
||||
model: "image-deployment",
|
||||
prompt: "draw a clean product render",
|
||||
cfg: buildConfig(),
|
||||
size: "768x1365",
|
||||
timeoutMs: 12_345,
|
||||
ssrfPolicy: { allowDocumentationRanges: true },
|
||||
});
|
||||
|
||||
expect(resolveApiKeyForProviderMock).toHaveBeenCalledWith({
|
||||
provider: PROVIDER_ID,
|
||||
cfg: buildConfig(),
|
||||
agentDir: undefined,
|
||||
store: undefined,
|
||||
});
|
||||
expect(resolveProviderHttpRequestConfigMock).toHaveBeenCalledWith({
|
||||
baseUrl: "https://example.services.ai.azure.com/mai/v1",
|
||||
defaultBaseUrl: "https://example.services.ai.azure.com/mai/v1",
|
||||
allowPrivateNetwork: false,
|
||||
defaultHeaders: { "api-key": "foundry-key" },
|
||||
request: undefined,
|
||||
provider: PROVIDER_ID,
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
});
|
||||
expect(postJsonRequestMock).toHaveBeenCalledOnce();
|
||||
const request = requirePostJsonRequest();
|
||||
expect(request.url).toBe("https://example.services.ai.azure.com/mai/v1/images/generations");
|
||||
expect(request.body).toEqual({
|
||||
model: "image-deployment",
|
||||
prompt: "draw a clean product render",
|
||||
width: 768,
|
||||
height: 1365,
|
||||
});
|
||||
expect(Object.fromEntries(requireHeaders(request.headers).entries())).toEqual({
|
||||
"api-key": "foundry-key",
|
||||
"content-type": "application/json",
|
||||
});
|
||||
expect(request.timeoutMs).toBe(12_345);
|
||||
expect(request.ssrfPolicy).toEqual({ allowDocumentationRanges: true });
|
||||
expect(result.model).toBe("image-deployment");
|
||||
expect(result.images[0]?.buffer.toString()).toBe("png");
|
||||
expect(result.images[0]?.mimeType).toBe("image/png");
|
||||
});
|
||||
|
||||
it("refreshes Entra ID auth and sends MAI image edits as multipart form data", async () => {
|
||||
resolveApiKeyForProviderMock.mockResolvedValueOnce({
|
||||
apiKey: "__entra_id_dynamic__",
|
||||
mode: "api-key",
|
||||
profileId: "microsoft-foundry:entra",
|
||||
source: "profile:microsoft-foundry:entra",
|
||||
});
|
||||
prepareFoundryRuntimeAuthMock.mockResolvedValueOnce({
|
||||
apiKey: "entra-token",
|
||||
baseUrl: "https://example.services.ai.azure.com/openai/v1",
|
||||
expiresAt: Date.now() + 60_000,
|
||||
});
|
||||
postMultipartRequestMock.mockResolvedValue(
|
||||
releasedJson({
|
||||
data: [{ b64_json: Buffer.from("edited").toString("base64") }],
|
||||
}),
|
||||
);
|
||||
const provider = buildMicrosoftFoundryImageGenerationProvider();
|
||||
|
||||
const result = await provider.generateImage({
|
||||
provider: PROVIDER_ID,
|
||||
model: "image-deployment",
|
||||
prompt: "make it brighter",
|
||||
cfg: buildConfig(),
|
||||
agentDir: "/agent",
|
||||
inputImages: [
|
||||
{
|
||||
buffer: Buffer.from("input"),
|
||||
mimeType: "image/png",
|
||||
fileName: "input.png",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(prepareFoundryRuntimeAuthMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
agentDir: "/agent",
|
||||
provider: PROVIDER_ID,
|
||||
modelId: "image-deployment",
|
||||
apiKey: "__entra_id_dynamic__",
|
||||
authMode: "api-key",
|
||||
profileId: "microsoft-foundry:entra",
|
||||
}),
|
||||
);
|
||||
expect(postMultipartRequestMock).toHaveBeenCalledOnce();
|
||||
const request = requirePostMultipartRequest();
|
||||
expect(request.url).toBe("https://example.services.ai.azure.com/mai/v1/images/edits");
|
||||
expect(Object.fromEntries(requireHeaders(request.headers).entries())).toEqual({
|
||||
authorization: "Bearer entra-token",
|
||||
});
|
||||
const form = request.body as FormData;
|
||||
expect(form.get("model")).toBe("image-deployment");
|
||||
expect(form.get("prompt")).toBe("make it brighter");
|
||||
const image = form.get("image");
|
||||
expect(image).toBeInstanceOf(Blob);
|
||||
expect((image as File).name).toBe("input.png");
|
||||
expect((image as File).type).toBe("image/png");
|
||||
expect(result.images[0]?.buffer.toString()).toBe("edited");
|
||||
});
|
||||
|
||||
it("rejects image edits for MAI text-to-image-only deployments", async () => {
|
||||
const provider = buildMicrosoftFoundryImageGenerationProvider();
|
||||
|
||||
await expect(
|
||||
provider.generateImage({
|
||||
provider: PROVIDER_ID,
|
||||
model: "image-deployment",
|
||||
prompt: "edit it",
|
||||
cfg: buildConfig({ modelName: "MAI-Image-2e" }),
|
||||
inputImages: [{ buffer: Buffer.from("input"), mimeType: "image/png" }],
|
||||
}),
|
||||
).rejects.toThrow("MAI-Image-2e does not support Microsoft Foundry MAI image edits.");
|
||||
expect(resolveApiKeyForProviderMock).not.toHaveBeenCalled();
|
||||
expect(postMultipartRequestMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects non-MAI image deployments before making requests", async () => {
|
||||
const provider = buildMicrosoftFoundryImageGenerationProvider();
|
||||
|
||||
await expect(
|
||||
provider.generateImage({
|
||||
provider: PROVIDER_ID,
|
||||
model: "gpt-deployment",
|
||||
prompt: "draw it",
|
||||
cfg: buildConfig({ modelId: "gpt-deployment", modelName: "gpt-5.4" }),
|
||||
}),
|
||||
).rejects.toThrow('supports MAI image deployments only, got "gpt-5.4"');
|
||||
expect(resolveApiKeyForProviderMock).not.toHaveBeenCalled();
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects MAI image sizes outside Microsoft Foundry limits", async () => {
|
||||
const provider = buildMicrosoftFoundryImageGenerationProvider();
|
||||
|
||||
await expect(
|
||||
provider.generateImage({
|
||||
provider: PROVIDER_ID,
|
||||
model: "image-deployment",
|
||||
prompt: "draw it",
|
||||
cfg: buildConfig(),
|
||||
size: "512x512",
|
||||
}),
|
||||
).rejects.toThrow("at least 768x768");
|
||||
expect(postJsonRequestMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,348 @@
|
||||
// Microsoft Foundry image provider routes MAI image deployments to the MAI API.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { ProviderRuntimeModel } from "openclaw/plugin-sdk/core";
|
||||
import type {
|
||||
ImageGenerationProvider,
|
||||
ImageGenerationRequest,
|
||||
ImageGenerationResult,
|
||||
ImageGenerationSourceImage,
|
||||
} from "openclaw/plugin-sdk/image-generation";
|
||||
import {
|
||||
imageSourceUploadFileName,
|
||||
parseOpenAiCompatibleImageResponse,
|
||||
} from "openclaw/plugin-sdk/image-generation";
|
||||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
postJsonRequest,
|
||||
postMultipartRequest,
|
||||
resolveProviderHttpRequestConfig,
|
||||
resolveProviderOperationTimeoutMs,
|
||||
sanitizeConfiguredModelProviderRequest,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { prepareFoundryRuntimeAuth } from "./runtime.js";
|
||||
import { extractFoundryEndpoint } from "./shared-runtime.js";
|
||||
import {
|
||||
DEFAULT_API,
|
||||
isFoundryMaiImageModel,
|
||||
isFoundryProviderApi,
|
||||
MAI_DEFAULT_IMAGE_MODEL,
|
||||
MAI_IMAGE_MODELS,
|
||||
PROVIDER_ID,
|
||||
} from "./shared.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 600_000;
|
||||
const DEFAULT_IMAGE_SIZE = { width: 1024, height: 1024 };
|
||||
const MAI_MIN_IMAGE_SIDE_PX = 768;
|
||||
const MAI_MAX_IMAGE_PIXELS = 1_048_576;
|
||||
const MAI_IMAGE_BASE_PATH = "/mai/v1";
|
||||
const MAI_IMAGE_OUTPUT_MIME = "image/png";
|
||||
const MAI_IMAGE_UPLOAD_MIME_TYPES = new Set(["image/jpeg", "image/jpg", "image/png"]);
|
||||
|
||||
type ModelProviderConfig = NonNullable<NonNullable<OpenClawConfig["models"]>["providers"]>[string];
|
||||
|
||||
function readProviderConfig(req: ImageGenerationRequest): ModelProviderConfig | undefined {
|
||||
return req.cfg.models?.providers?.[PROVIDER_ID];
|
||||
}
|
||||
|
||||
function resolveConfiguredModelName(
|
||||
providerConfig: ModelProviderConfig | undefined,
|
||||
model: string,
|
||||
): string {
|
||||
return providerConfig?.models.find((candidate) => candidate.id === model)?.name ?? model;
|
||||
}
|
||||
|
||||
function ensureMaiImageModel(
|
||||
providerConfig: ModelProviderConfig | undefined,
|
||||
model: string,
|
||||
): string {
|
||||
const modelName = resolveConfiguredModelName(providerConfig, model);
|
||||
if (!isFoundryMaiImageModel(modelName)) {
|
||||
throw new Error(
|
||||
`Microsoft Foundry image generation supports MAI image deployments only, got "${modelName}".`,
|
||||
);
|
||||
}
|
||||
return modelName;
|
||||
}
|
||||
|
||||
function isMaiImageEditModel(modelName: string): boolean {
|
||||
const normalized = normalizeOptionalLowercaseString(modelName);
|
||||
return normalized === "mai-image-2.5" || normalized === "mai-image-2.5-flash";
|
||||
}
|
||||
|
||||
function resolveMaiImageSize(size: string | undefined): { width: number; height: number } {
|
||||
if (!size) {
|
||||
return DEFAULT_IMAGE_SIZE;
|
||||
}
|
||||
const match = size.match(/^(\d{1,5})x(\d{1,5})$/u);
|
||||
if (!match) {
|
||||
throw new Error(`Microsoft Foundry MAI image size must use WIDTHxHEIGHT, got "${size}".`);
|
||||
}
|
||||
const width = Number(match[1]);
|
||||
const height = Number(match[2]);
|
||||
if (
|
||||
!Number.isInteger(width) ||
|
||||
!Number.isInteger(height) ||
|
||||
width < MAI_MIN_IMAGE_SIDE_PX ||
|
||||
height < MAI_MIN_IMAGE_SIDE_PX ||
|
||||
width * height > MAI_MAX_IMAGE_PIXELS
|
||||
) {
|
||||
throw new Error(
|
||||
`Microsoft Foundry MAI image size must be at least 768x768 and at most 1,048,576 total pixels, got "${size}".`,
|
||||
);
|
||||
}
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function assertSingleImageCount(count: number | undefined): void {
|
||||
if (count === undefined || count === 1) {
|
||||
return;
|
||||
}
|
||||
throw new Error("Microsoft Foundry MAI image models return one image per request.");
|
||||
}
|
||||
|
||||
function resolveConfiguredEndpoint(params: {
|
||||
providerConfig: ModelProviderConfig | undefined;
|
||||
preparedBaseUrl?: string;
|
||||
}): string {
|
||||
const endpoint = extractFoundryEndpoint(params.preparedBaseUrl ?? params.providerConfig?.baseUrl);
|
||||
if (!endpoint) {
|
||||
throw new Error("Microsoft Foundry endpoint missing for MAI image generation.");
|
||||
}
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
function buildMaiImageUrl(baseUrl: string, mode: "generations" | "edits"): string {
|
||||
return `${baseUrl.replace(/\/+$/u, "")}/images/${mode}`;
|
||||
}
|
||||
|
||||
function buildRuntimeModel(params: {
|
||||
providerConfig: ModelProviderConfig | undefined;
|
||||
model: string;
|
||||
modelName: string;
|
||||
}): ProviderRuntimeModel {
|
||||
const api = isFoundryProviderApi(params.providerConfig?.api)
|
||||
? params.providerConfig.api
|
||||
: DEFAULT_API;
|
||||
return {
|
||||
id: params.model,
|
||||
name: params.modelName,
|
||||
api,
|
||||
provider: PROVIDER_ID,
|
||||
baseUrl: params.providerConfig?.baseUrl ?? "",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 32_000,
|
||||
maxTokens: 0,
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveMaiImageAuth(params: {
|
||||
req: ImageGenerationRequest;
|
||||
providerConfig: ModelProviderConfig | undefined;
|
||||
model: string;
|
||||
modelName: string;
|
||||
}): Promise<{ headers: Record<string, string>; baseUrl?: string }> {
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: PROVIDER_ID,
|
||||
cfg: params.req.cfg,
|
||||
agentDir: params.req.agentDir,
|
||||
store: params.req.authStore,
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("Microsoft Foundry API key missing");
|
||||
}
|
||||
if (auth.apiKey !== "__entra_id_dynamic__") {
|
||||
return {
|
||||
headers: {
|
||||
"api-key": auth.apiKey,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const prepared = await prepareFoundryRuntimeAuth({
|
||||
config: params.req.cfg,
|
||||
agentDir: params.req.agentDir,
|
||||
env: process.env,
|
||||
provider: PROVIDER_ID,
|
||||
modelId: params.model,
|
||||
model: buildRuntimeModel({
|
||||
providerConfig: params.providerConfig,
|
||||
model: params.model,
|
||||
modelName: params.modelName,
|
||||
}),
|
||||
apiKey: auth.apiKey,
|
||||
authMode: auth.mode,
|
||||
...(auth.profileId ? { profileId: auth.profileId } : {}),
|
||||
});
|
||||
if (!prepared?.apiKey) {
|
||||
throw new Error("Microsoft Foundry Entra ID token missing after runtime auth refresh.");
|
||||
}
|
||||
return {
|
||||
headers: {
|
||||
Authorization: `Bearer ${prepared.apiKey}`,
|
||||
},
|
||||
...(prepared.baseUrl ? { baseUrl: prepared.baseUrl } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function buildEditFormData(params: {
|
||||
req: ImageGenerationRequest;
|
||||
image: ImageGenerationSourceImage;
|
||||
model: string;
|
||||
}): FormData {
|
||||
const mimeType = normalizeOptionalLowercaseString(params.image.mimeType) ?? MAI_IMAGE_OUTPUT_MIME;
|
||||
if (!MAI_IMAGE_UPLOAD_MIME_TYPES.has(mimeType)) {
|
||||
throw new Error("Microsoft Foundry MAI image edits require a PNG or JPEG input image.");
|
||||
}
|
||||
const form = new FormData();
|
||||
form.set("model", params.model);
|
||||
form.set("prompt", params.req.prompt);
|
||||
form.set(
|
||||
"image",
|
||||
new Blob([new Uint8Array(params.image.buffer)], {
|
||||
type: mimeType === "image/jpg" ? "image/jpeg" : mimeType,
|
||||
}),
|
||||
imageSourceUploadFileName({
|
||||
image: params.image,
|
||||
index: 0,
|
||||
fileNamePrefix: "microsoft-foundry-input",
|
||||
}),
|
||||
);
|
||||
return form;
|
||||
}
|
||||
|
||||
function parseMaiImageResponse(payload: unknown, label: string) {
|
||||
const images = parseOpenAiCompatibleImageResponse(payload, {
|
||||
defaultMimeType: MAI_IMAGE_OUTPUT_MIME,
|
||||
fileNamePrefix: "microsoft-foundry-image",
|
||||
malformedResponseError: `${label} response malformed`,
|
||||
sniffMimeType: true,
|
||||
});
|
||||
if (images.length === 0) {
|
||||
throw new Error(`${label} response missing image data`);
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
export function buildMicrosoftFoundryImageGenerationProvider(): ImageGenerationProvider {
|
||||
return {
|
||||
id: PROVIDER_ID,
|
||||
label: "Microsoft Foundry",
|
||||
defaultModel: MAI_DEFAULT_IMAGE_MODEL,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
models: [...MAI_IMAGE_MODELS],
|
||||
isConfigured: ({ agentDir }) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: PROVIDER_ID,
|
||||
agentDir,
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 1,
|
||||
supportsSize: true,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 1,
|
||||
maxInputImages: 1,
|
||||
supportsSize: false,
|
||||
},
|
||||
geometry: {
|
||||
sizes: ["1024x1024", "768x1365", "1365x768"],
|
||||
},
|
||||
output: {
|
||||
formats: ["png"],
|
||||
},
|
||||
},
|
||||
async generateImage(req): Promise<ImageGenerationResult> {
|
||||
const providerConfig = readProviderConfig(req);
|
||||
const model = req.model || MAI_DEFAULT_IMAGE_MODEL;
|
||||
const modelName = ensureMaiImageModel(providerConfig, model);
|
||||
const inputImages = req.inputImages ?? [];
|
||||
const mode = inputImages.length > 0 ? "edits" : "generations";
|
||||
assertSingleImageCount(req.count);
|
||||
if (inputImages.length > 1) {
|
||||
throw new Error("Microsoft Foundry MAI image edits support one input image.");
|
||||
}
|
||||
if (mode === "edits" && !isMaiImageEditModel(modelName)) {
|
||||
throw new Error(`${modelName} does not support Microsoft Foundry MAI image edits.`);
|
||||
}
|
||||
|
||||
const auth = await resolveMaiImageAuth({ req, providerConfig, model, modelName });
|
||||
const endpoint = resolveConfiguredEndpoint({
|
||||
providerConfig,
|
||||
preparedBaseUrl: auth.baseUrl,
|
||||
});
|
||||
const resolvedBaseUrl = `${endpoint}${MAI_IMAGE_BASE_PATH}`;
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveProviderHttpRequestConfig({
|
||||
baseUrl: resolvedBaseUrl,
|
||||
defaultBaseUrl: resolvedBaseUrl,
|
||||
allowPrivateNetwork: false,
|
||||
defaultHeaders: auth.headers,
|
||||
request: sanitizeConfiguredModelProviderRequest(providerConfig?.request),
|
||||
provider: PROVIDER_ID,
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
});
|
||||
const timeoutMs = resolveProviderOperationTimeoutMs({
|
||||
timeoutMs: req.timeoutMs,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
const request =
|
||||
mode === "edits"
|
||||
? postMultipartRequest({
|
||||
url: buildMaiImageUrl(baseUrl, mode),
|
||||
headers,
|
||||
body: buildEditFormData({
|
||||
req,
|
||||
image: inputImages[0] as ImageGenerationSourceImage,
|
||||
model,
|
||||
}),
|
||||
timeoutMs,
|
||||
fetchFn: fetch,
|
||||
allowPrivateNetwork,
|
||||
ssrfPolicy: req.ssrfPolicy,
|
||||
dispatcherPolicy,
|
||||
})
|
||||
: postJsonRequest({
|
||||
url: buildMaiImageUrl(baseUrl, mode),
|
||||
headers: (() => {
|
||||
const jsonHeaders = new Headers(headers);
|
||||
jsonHeaders.set("Content-Type", "application/json");
|
||||
return jsonHeaders;
|
||||
})(),
|
||||
body: {
|
||||
model,
|
||||
prompt: req.prompt,
|
||||
...resolveMaiImageSize(req.size),
|
||||
},
|
||||
timeoutMs,
|
||||
fetchFn: fetch,
|
||||
allowPrivateNetwork,
|
||||
ssrfPolicy: req.ssrfPolicy,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
|
||||
const { response, release } = await request;
|
||||
try {
|
||||
const label =
|
||||
mode === "edits"
|
||||
? "Microsoft Foundry MAI image edit"
|
||||
: "Microsoft Foundry MAI image generation";
|
||||
await assertOkOrThrowHttpError(response, `${label} failed`);
|
||||
return {
|
||||
images: parseMaiImageResponse(await response.json(), label),
|
||||
model,
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
// Microsoft Foundry plugin entrypoint registers its OpenClaw integration.
|
||||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { buildMicrosoftFoundryImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import { buildMicrosoftFoundryProvider } from "./provider.js";
|
||||
|
||||
export default definePluginEntry({
|
||||
@@ -8,5 +9,6 @@ export default definePluginEntry({
|
||||
description: "Microsoft Foundry provider with Entra ID and API key auth",
|
||||
register(api) {
|
||||
api.registerProvider(buildMicrosoftFoundryProvider());
|
||||
api.registerImageGenerationProvider(buildMicrosoftFoundryImageGenerationProvider());
|
||||
},
|
||||
});
|
||||
|
||||
@@ -13,6 +13,18 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"contracts": {
|
||||
"imageGenerationProviders": ["microsoft-foundry"]
|
||||
},
|
||||
"imageGenerationProviderMetadata": {
|
||||
"microsoft-foundry": {
|
||||
"authSignals": [
|
||||
{
|
||||
"provider": "microsoft-foundry"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"providerAuthChoices": [
|
||||
{
|
||||
"provider": "microsoft-foundry",
|
||||
|
||||
Reference in New Issue
Block a user