diff --git a/CHANGELOG.md b/CHANGELOG.md index 788adf73072d..d79c561c682e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Media: install Sharp with the root package and fall back to sips, Windows native imaging, ImageMagick, GraphicsMagick, or ffmpeg for image resizing/conversion when Sharp is unavailable. Fixes #83401. Thanks @scotthuang. - Channels/bundled: append `openclaw doctor --fix` guidance to the bundled-channel load warnings emitted on `ERR_MODULE_NOT_FOUND` / `MODULE_NOT_FOUND` (including those wrapped on `.cause` by the native-require loader), so users hitting unstaged plugin runtime deps (e.g. `nostr-tools`) see an actionable repair hint instead of a bare module-not-found warning. (#76974) Thanks @BSG2000. - Telegram: deliver generated media completions back into forum topics by preserving topic IDs across requester-agent handoff. (#83556) Thanks @fuller-stack-dev. - Gateway: defer update-check startup until after readiness so package update checks no longer block sidecar-ready startup, while preserving update broadcasts and shutdown cleanup. (#83520) Thanks @samzong. diff --git a/extensions/browser/src/browser/screenshot.test.ts b/extensions/browser/src/browser/screenshot.test.ts index 7add1ee1d8b0..f113dc5c8b5b 100644 --- a/extensions/browser/src/browser/screenshot.test.ts +++ b/extensions/browser/src/browser/screenshot.test.ts @@ -3,6 +3,22 @@ import { describe, expect, it } from "vitest"; import { normalizeBrowserScreenshot } from "./screenshot.js"; describe("browser screenshot normalization", () => { + const unavailableImageBackend = process.platform === "win32" ? "sips" : "windows-native"; + + async function withUnavailableImageBackend(fn: () => Promise): Promise { + const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND; + process.env.OPENCLAW_IMAGE_BACKEND = unavailableImageBackend; + try { + return await fn(); + } finally { + if (previousBackend === undefined) { + delete process.env.OPENCLAW_IMAGE_BACKEND; + } else { + process.env.OPENCLAW_IMAGE_BACKEND = previousBackend; + } + } + } + it("shrinks oversized images to <=2000x2000 and <=5MB", async () => { const bigPng = await sharp({ create: { @@ -47,4 +63,27 @@ describe("browser screenshot normalization", () => { expect(normalized.buffer.equals(jpeg)).toBe(true); }); + + it("rejects screenshots above max side when no image processor is available", async () => { + const png = await sharp({ + create: { + width: 420, + height: 120, + channels: 3, + background: { r: 12, g: 34, b: 56 }, + }, + }) + .png({ compressionLevel: 9 }) + .toBuffer(); + expect(png.byteLength).toBeLessThan(5 * 1024 * 1024); + + await withUnavailableImageBackend(async () => { + await expect( + normalizeBrowserScreenshot(png, { + maxSide: 120, + maxBytes: 5 * 1024 * 1024, + }), + ).rejects.toThrow(/image processor unavailable/i); + }); + }); }); diff --git a/extensions/browser/src/browser/screenshot.ts b/extensions/browser/src/browser/screenshot.ts index 901eb0f77f83..dbccbd1cfe0e 100644 --- a/extensions/browser/src/browser/screenshot.ts +++ b/extensions/browser/src/browser/screenshot.ts @@ -2,8 +2,9 @@ import { buildImageResizeSideGrid, getImageMetadata, IMAGE_REDUCE_QUALITY_STEPS, + isImageProcessorUnavailableError, resizeToJpeg, -} from "../media/image-ops.js"; +} from "../media/media-services.js"; export const DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE = 2000; export const DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES = 5 * 1024 * 1024; @@ -31,15 +32,25 @@ export async function normalizeBrowserScreenshot( const sideGrid = buildImageResizeSideGrid(maxSide, sideStart); let smallest: { buffer: Buffer; size: number } | null = null; + let processorUnavailableError: unknown; for (const side of sideGrid) { for (const quality of IMAGE_REDUCE_QUALITY_STEPS) { - const out = await resizeToJpeg({ - buffer, - maxSide: side, - quality, - withoutEnlargement: true, - }); + let out: Buffer; + try { + out = await resizeToJpeg({ + buffer, + maxSide: side, + quality, + withoutEnlargement: true, + }); + } catch (err) { + if (isImageProcessorUnavailableError(err)) { + processorUnavailableError = err; + break; + } + throw err; + } if (!smallest || out.byteLength < smallest.size) { smallest = { buffer: out, size: out.byteLength }; @@ -49,6 +60,13 @@ export async function normalizeBrowserScreenshot( return { buffer: out, contentType: "image/jpeg" }; } } + if (processorUnavailableError) { + break; + } + } + + if (processorUnavailableError) { + throw processorUnavailableError; } const best = smallest?.buffer ?? buffer; diff --git a/extensions/browser/src/media/image-ops.ts b/extensions/browser/src/media/image-ops.ts index b8024a6757c0..556f242f0f00 100644 --- a/extensions/browser/src/media/image-ops.ts +++ b/extensions/browser/src/media/image-ops.ts @@ -3,4 +3,4 @@ export { buildImageResizeSideGrid, getImageMetadata, resizeToJpeg, -} from "../sdk-setup-tools.js"; +} from "./media-services.js"; diff --git a/extensions/browser/src/media/media-services.ts b/extensions/browser/src/media/media-services.ts new file mode 100644 index 000000000000..a92f1319dab9 --- /dev/null +++ b/extensions/browser/src/media/media-services.ts @@ -0,0 +1,7 @@ +export { + IMAGE_REDUCE_QUALITY_STEPS, + buildImageResizeSideGrid, + getImageMetadata, + isImageProcessorUnavailableError, + resizeToJpeg, +} from "../sdk-setup-tools.js"; diff --git a/extensions/browser/src/sdk-setup-tools.ts b/extensions/browser/src/sdk-setup-tools.ts index 03fc423c38bc..2b229c095521 100644 --- a/extensions/browser/src/sdk-setup-tools.ts +++ b/extensions/browser/src/sdk-setup-tools.ts @@ -23,6 +23,7 @@ export { IMAGE_REDUCE_QUALITY_STEPS, buildImageResizeSideGrid, getImageMetadata, + isImageProcessorUnavailableError, resizeToJpeg, } from "openclaw/plugin-sdk/media-runtime"; export { detectMime } from "openclaw/plugin-sdk/media-mime"; diff --git a/extensions/imessage/src/monitor/media-staging.ts b/extensions/imessage/src/monitor/media-staging.ts index 08db64d2c0ed..6e7bfcaccee5 100644 --- a/extensions/imessage/src/monitor/media-staging.ts +++ b/extensions/imessage/src/monitor/media-staging.ts @@ -1,17 +1,10 @@ -import { execFile } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; -import { promisify } from "node:util"; import { isInboundPathAllowed } from "openclaw/plugin-sdk/media-runtime"; import { saveMediaBuffer } from "openclaw/plugin-sdk/media-store"; -import { buildRandomTempFilePath } from "openclaw/plugin-sdk/temp-path"; +import { loadWebMedia } from "openclaw/plugin-sdk/web-media"; import type { IMessageAttachment } from "./types.js"; -const execFileAsync = promisify(execFile); - -const HEIC_CONVERSION_TIMEOUT_MS = 15_000; -const HEIC_CONVERSION_MAX_BUFFER_BYTES = 64 * 1024; - export type StagedIMessageAttachment = { path: string; contentType?: string; @@ -73,43 +66,6 @@ async function resolveAllowedCanonicalAttachmentPath(params: { return canonicalPath; } -async function convertHeicToJpegWithSips(sourcePath: string, maxBytes: number): Promise { - const tempPath = buildRandomTempFilePath({ - prefix: "openclaw-imessage", - extension: "jpg", - }); - try { - await execFileAsync( - "sips", - [ - "-s", - "format", - "jpeg", - "-s", - "formatOptions", - "90", - "-Z", - "4096", - sourcePath, - "--out", - tempPath, - ], - { - timeout: HEIC_CONVERSION_TIMEOUT_MS, - maxBuffer: HEIC_CONVERSION_MAX_BUFFER_BYTES, - killSignal: "SIGKILL", - }, - ); - const stat = await fs.stat(tempPath); - if (stat.size > maxBytes) { - throw new Error(`converted media exceeds ${Math.round(maxBytes / (1024 * 1024))}MB limit`); - } - return await fs.readFile(tempPath); - } finally { - await fs.rm(tempPath, { force: true }).catch(() => {}); - } -} - async function readAttachmentBuffer(params: { attachmentPath: string; mimeType?: string | null; @@ -142,11 +98,20 @@ async function readAttachmentBuffer(params: { if (isHeicAttachment(params.attachmentPath, params.mimeType)) { try { - const convert = params.deps.convertHeicToJpeg ?? convertHeicToJpegWithSips; + const convert = params.deps.convertHeicToJpeg; + const converted = convert + ? { + buffer: await convert(canonicalPath, params.maxBytes), + fileName: jpegFilenameForAttachment(params.attachmentPath), + } + : await loadWebMedia(canonicalPath, { + maxBytes: params.maxBytes, + localRoots: [path.dirname(canonicalPath)], + }); return { - buffer: await convert(canonicalPath, params.maxBytes), + buffer: converted.buffer, contentType: "image/jpeg", - originalFilename: jpegFilenameForAttachment(params.attachmentPath), + originalFilename: converted.fileName ?? jpegFilenameForAttachment(params.attachmentPath), }; } catch (err) { params.deps.logVerbose?.( diff --git a/extensions/speech-core/src/audio-transcode.test.ts b/extensions/speech-core/src/audio-transcode.test.ts deleted file mode 100644 index 886bd09675e6..000000000000 --- a/extensions/speech-core/src/audio-transcode.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { transcodeAudioBuffer } from "./audio-transcode.js"; - -describe("transcodeAudioBuffer", () => { - it("returns noop-same-container when source and target containers match", async () => { - const result = await transcodeAudioBuffer({ - audioBuffer: Buffer.from("payload"), - sourceExtension: "mp3", - targetExtension: ".mp3", - }); - expect(result).toEqual({ ok: false, reason: "noop-same-container" }); - }); - - it("returns no-recipe when no afconvert recipe is defined for the requested pair", async () => { - const result = await transcodeAudioBuffer({ - audioBuffer: Buffer.from("payload"), - sourceExtension: "mp3", - targetExtension: "flac", - }); - expect(result).toEqual({ ok: false, reason: "no-recipe" }); - }); - - it("returns invalid-extension for an empty source extension", async () => { - const result = await transcodeAudioBuffer({ - audioBuffer: Buffer.from("payload"), - sourceExtension: "", - targetExtension: "caf", - }); - expect(result).toEqual({ ok: false, reason: "invalid-extension" }); - }); - - it("returns invalid-extension for an empty target extension", async () => { - const result = await transcodeAudioBuffer({ - audioBuffer: Buffer.from("payload"), - sourceExtension: "mp3", - targetExtension: "", - }); - expect(result).toEqual({ ok: false, reason: "invalid-extension" }); - }); - - it("rejects path-traversal style extensions", async () => { - const result = await transcodeAudioBuffer({ - audioBuffer: Buffer.from("payload"), - sourceExtension: "../etc/passwd", - targetExtension: "caf", - }); - expect(result).toEqual({ ok: false, reason: "invalid-extension" }); - }); - - it("returns platform-unsupported off-Darwin without invoking afconvert", async () => { - if (process.platform === "darwin") { - // macOS: a valid mp3→caf request would proceed to spawn `afconvert`, - // which we don't want to run from a unit test. The Darwin happy path - // is exercised end-to-end via the native voice-memo flow. - return; - } - const result = await transcodeAudioBuffer({ - audioBuffer: Buffer.from("payload"), - sourceExtension: "mp3", - targetExtension: "caf", - }); - expect(result).toEqual({ ok: false, reason: "platform-unsupported" }); - }); -}); diff --git a/extensions/speech-core/src/audio-transcode.ts b/extensions/speech-core/src/audio-transcode.ts deleted file mode 100644 index 3835574f0e65..000000000000 --- a/extensions/speech-core/src/audio-transcode.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { spawn } from "node:child_process"; -import { tempWorkspaceSync, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox"; - -type TranscodeOutcome = - | { ok: true; buffer: Buffer } - | { - ok: false; - reason: - | "platform-unsupported" - | "invalid-extension" - | "noop-same-container" - | "no-recipe" - | "transcoder-failed"; - detail?: string; - }; - -/** - * Best-effort audio container transcode using macOS `afconvert`. - * - * Used by the TTS pipeline to pre-encode synthesized audio into a channel's - * preferred container (see `ChannelTtsVoiceDeliveryCapabilities.preferAudioFileFormat`) - * so the channel's downstream does not have to perform a container - * conversion of its own. Returns a discriminated outcome so callers can - * distinguish "we didn't try" (platform/recipe/noop) from "we tried and the - * transcoder failed", which is the case worth logging. - * - * Currently only macOS is supported because `afconvert` is the only widely - * available encoder we ship a recipe for. - */ -export async function transcodeAudioBuffer(params: { - audioBuffer: Buffer; - sourceExtension: string; - targetExtension: string; - timeoutMs?: number; -}): Promise { - // Validate inputs first so callers get a specific reason regardless of - // host platform. Platform-unsupported is the gate immediately before the - // actual `afconvert` invocation. - const source = normalizeExt(params.sourceExtension); - const target = normalizeExt(params.targetExtension); - if (!source || !target) { - return { ok: false, reason: "invalid-extension" }; - } - if (source === target) { - return { ok: false, reason: "noop-same-container" }; - } - const recipe = pickAfconvertRecipe(source, target); - if (!recipe) { - return { ok: false, reason: "no-recipe" }; - } - if (process.platform !== "darwin") { - return { ok: false, reason: "platform-unsupported" }; - } - - const tmp = tempWorkspaceSync({ - rootDir: resolvePreferredOpenClawTmpDir(), - prefix: "tts-transcode-", - }); - const inPath = tmp.write(`in.${source}`, params.audioBuffer); - const outPath = tmp.path(`out.${target}`); - try { - const result = await runAfconvert({ - args: [...recipe, inPath, outPath], - timeoutMs: params.timeoutMs ?? 5000, - }); - if (!result.ok) { - return { ok: false, reason: "transcoder-failed", detail: result.detail }; - } - return { ok: true, buffer: tmp.read(`out.${target}`) }; - } catch (err) { - return { ok: false, reason: "transcoder-failed", detail: (err as Error).message }; - } finally { - tmp.cleanup(); - } -} - -function normalizeExt(ext: string): string | undefined { - // Pattern matches the sibling helper in src/media/audio-transcode.ts: a short - // alphanumeric extension token. Keeps the value safe to interpolate into - // tmp-file names below without introducing a path-traversal surface. - const trimmed = ext.trim().toLowerCase().replace(/^\./, ""); - return /^[a-z0-9]{1,12}$/.test(trimmed) ? trimmed : undefined; -} - -function pickAfconvertRecipe(_source: string, target: string): string[] | undefined { - // Currently only the MP3->CAF path used by native Messages voice memos. - if (target === "caf") { - // Opus-in-CAF, mono, 24 kHz. Validated against macOS 15.x Messages.app's - // native voice-memo CAF descriptor (1 ch, 24000 Hz, opus); other CAF - // flavors (PCM, AAC) get downgraded to plain audio attachments along the - // Messages.app path. If iMessage stops rendering the result - // as a voice memo after a system update, try forcing frames-per-packet - // explicitly via `opus@24000#480` and re-validate. See #72506. - return ["-f", "caff", "-d", "opus@24000", "-c", "1"]; - } - return undefined; -} - -function runAfconvert(params: { - args: string[]; - timeoutMs: number; -}): Promise<{ ok: true } | { ok: false; detail: string }> { - return new Promise((resolve) => { - const child = spawn("/usr/bin/afconvert", params.args, { stdio: "ignore" }); - const timer = setTimeout(() => { - child.kill("SIGKILL"); - resolve({ ok: false, detail: `timeout-${params.timeoutMs}ms` }); - }, params.timeoutMs); - child.once("error", (err) => { - clearTimeout(timer); - resolve({ ok: false, detail: err.message }); - }); - child.once("exit", (code) => { - clearTimeout(timer); - if (code === 0) { - resolve({ ok: true }); - } else { - resolve({ ok: false, detail: `exit-${code ?? "unknown"}` }); - } - }); - }); -} diff --git a/extensions/speech-core/src/tts.test.ts b/extensions/speech-core/src/tts.test.ts index 154849642db3..85ba8e11a1a4 100644 --- a/extensions/speech-core/src/tts.test.ts +++ b/extensions/speech-core/src/tts.test.ts @@ -55,7 +55,7 @@ const transcodeAudioBufferMock = vi.hoisted(() => >(async () => ({ ok: false, reason: "platform-unsupported" })), ); -vi.mock("./audio-transcode.js", () => ({ +vi.mock("openclaw/plugin-sdk/media-runtime", () => ({ transcodeAudioBuffer: transcodeAudioBufferMock, })); diff --git a/extensions/speech-core/src/tts.ts b/extensions/speech-core/src/tts.ts index a3836eedd278..5f75acf4ddcd 100644 --- a/extensions/speech-core/src/tts.ts +++ b/extensions/speech-core/src/tts.ts @@ -11,6 +11,7 @@ import type { } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core"; +import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime"; import { markReplyPayloadAsTtsSupplement, resolveSendableOutboundReplyParts, @@ -50,7 +51,6 @@ import { type TtsDirectiveParseResult, type TtsConfigResolutionContext, } from "../api.js"; -import { transcodeAudioBuffer } from "./audio-transcode.js"; export type { ResolvedTtsConfig, diff --git a/package.json b/package.json index 9aaa4d28557d..830173168af0 100644 --- a/package.json +++ b/package.json @@ -1837,6 +1837,7 @@ "vitest": "4.1.6" }, "optionalDependencies": { + "sharp": "0.34.5", "sqlite-vec": "0.1.9" }, "overrides": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a7846b05fc8d..4e2a565f72f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -262,6 +262,9 @@ importers: specifier: 4.1.6 version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@25.8.0)(@vitest/browser-playwright@4.1.6)(@vitest/coverage-v8@4.1.6)(jsdom@29.1.1(@noble/hashes@2.0.1))(vite@8.0.13(@types/node@25.8.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.0)(yaml@2.9.0)) optionalDependencies: + sharp: + specifier: 0.34.5 + version: 0.34.5 sqlite-vec: specifier: 0.1.9 version: 0.1.9 diff --git a/src/agents/openclaw-tools.camera.test.ts b/src/agents/openclaw-tools.camera.test.ts index 119362fb00d3..1eccbdcc6b91 100644 --- a/src/agents/openclaw-tools.camera.test.ts +++ b/src/agents/openclaw-tools.camera.test.ts @@ -10,7 +10,7 @@ const { callGateway } = vi.hoisted(() => ({ })); vi.mock("../gateway/call.js", () => ({ callGateway })); -vi.mock("../media/image-ops.js", () => ({ +vi.mock("../media/media-services.js", () => ({ getImageMetadata: vi.fn(async () => ({ width: 1, height: 1 })), resizeToJpeg: vi.fn(async () => Buffer.from("jpeg")), })); diff --git a/src/agents/tool-images.test.ts b/src/agents/tool-images.test.ts index 0c381df1f670..9e00a0d019b8 100644 --- a/src/agents/tool-images.test.ts +++ b/src/agents/tool-images.test.ts @@ -3,6 +3,22 @@ import { describe, expect, it } from "vitest"; import { sanitizeContentBlocksImages, sanitizeImageBlocks } from "./tool-images.js"; describe("tool image sanitizing", () => { + const unavailableImageBackend = process.platform === "win32" ? "sips" : "windows-native"; + + async function withUnavailableImageBackend(fn: () => Promise): Promise { + const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND; + process.env.OPENCLAW_IMAGE_BACKEND = unavailableImageBackend; + try { + return await fn(); + } finally { + if (previousBackend === undefined) { + delete process.env.OPENCLAW_IMAGE_BACKEND; + } else { + process.env.OPENCLAW_IMAGE_BACKEND = previousBackend; + } + } + } + const getImageBlock = ( blocks: Awaited>, ): (typeof blocks)[number] & { type: "image"; data: string; mimeType?: string } => { @@ -86,6 +102,29 @@ describe("tool image sanitizing", () => { expect(image.mimeType).toBe("image/jpeg"); }, 20_000); + it("drops images above max dimension when no image processor is available", async () => { + const png = await createWidePng(); + expect(png.byteLength).toBeLessThan(5 * 1024 * 1024); + + const blocks = [ + { + type: "image" as const, + data: png.toString("base64"), + mimeType: "image/png", + }, + ]; + + const out = await withUnavailableImageBackend(() => + sanitizeContentBlocksImages(blocks, "test", { maxDimensionPx: 120 }), + ); + + expect(out).toHaveLength(1); + expect(out[0].type).toBe("text"); + if (out[0].type === "text") { + expect(out[0].text).toMatch(/image processor unavailable/i); + } + }, 20_000); + it("corrects mismatched jpeg mimeType", async () => { const jpeg = await sharp({ create: { diff --git a/src/agents/tool-images.ts b/src/agents/tool-images.ts index 100efc025ec0..645d334513f6 100644 --- a/src/agents/tool-images.ts +++ b/src/agents/tool-images.ts @@ -6,8 +6,9 @@ import { buildImageResizeSideGrid, getImageMetadata, IMAGE_REDUCE_QUALITY_STEPS, + isImageProcessorUnavailableError, resizeToJpeg, -} from "../media/image-ops.js"; +} from "../media/media-services.js"; import { DEFAULT_IMAGE_MAX_BYTES, DEFAULT_IMAGE_MAX_DIMENSION_PX, @@ -187,14 +188,24 @@ async function resizeImageBase64IfNeeded(params: { const sideGrid = buildImageResizeSideGrid(params.maxDimensionPx, sideStart); let smallest: { buffer: Buffer; size: number } | null = null; + let processorUnavailableError: unknown; for (const side of sideGrid) { for (const quality of IMAGE_REDUCE_QUALITY_STEPS) { - const out = await resizeToJpeg({ - buffer: buf, - maxSide: side, - quality, - withoutEnlargement: true, - }); + let out: Buffer; + try { + out = await resizeToJpeg({ + buffer: buf, + maxSide: side, + quality, + withoutEnlargement: true, + }); + } catch (err) { + if (isImageProcessorUnavailableError(err)) { + processorUnavailableError = err; + break; + } + throw err; + } if (!smallest || out.byteLength < smallest.size) { smallest = { buffer: out, size: out.byteLength }; } @@ -239,6 +250,13 @@ async function resizeImageBase64IfNeeded(params: { }; } } + if (processorUnavailableError) { + break; + } + } + + if (processorUnavailableError) { + throw processorUnavailableError; } const best = smallest?.buffer ?? buf; diff --git a/src/agents/tools/image-generate-tool.test.ts b/src/agents/tools/image-generate-tool.test.ts index b61969e9e04c..6bf6fa90f59c 100644 --- a/src/agents/tools/image-generate-tool.test.ts +++ b/src/agents/tools/image-generate-tool.test.ts @@ -10,7 +10,7 @@ const taskRuntimeMocks = vi.hoisted(() => ({ vi.mock("../../tasks/detached-task-runtime.js", () => taskRuntimeMocks); let imageGenerationRuntime: typeof import("../../image-generation/runtime.js"); -let imageOps: typeof import("../../media/image-ops.js"); +let imageOps: typeof import("../../media/media-services.js"); let splitMediaFromOutput: typeof import("../../media/parse.js").splitMediaFromOutput; let mediaStore: typeof import("../../media/store.js"); let webMedia: typeof import("../../media/web-media.js"); @@ -288,7 +288,7 @@ describe("createImageGenerateTool", () => { }; }); imageGenerationRuntime = await import("../../image-generation/runtime.js"); - imageOps = await import("../../media/image-ops.js"); + imageOps = await import("../../media/media-services.js"); ({ splitMediaFromOutput } = await import("../../media/parse.js")); mediaStore = await import("../../media/store.js"); webMedia = await import("../../media/web-media.js"); diff --git a/src/agents/tools/image-generate-tool.ts b/src/agents/tools/image-generate-tool.ts index 481876764189..b9a9f4be745b 100644 --- a/src/agents/tools/image-generate-tool.ts +++ b/src/agents/tools/image-generate-tool.ts @@ -25,11 +25,11 @@ import { resolveConfiguredMediaMaxBytes, resolveGeneratedMediaMaxBytes, } from "../../media/configured-max-bytes.js"; -import { getImageMetadata } from "../../media/image-ops.js"; import { classifyMediaReferenceSource, normalizeMediaReferenceSource, } from "../../media/media-reference.js"; +import { getImageMetadata } from "../../media/media-services.js"; import { saveMediaBuffer } from "../../media/store.js"; import { loadWebMedia } from "../../media/web-media.js"; import { resolveUserPath } from "../../utils.js"; diff --git a/src/cli/capability-cli.test.ts b/src/cli/capability-cli.test.ts index 48982ded77a3..39f69542c9a1 100644 --- a/src/cli/capability-cli.test.ts +++ b/src/cli/capability-cli.test.ts @@ -242,12 +242,12 @@ vi.mock("../media-understanding/provider-registry.js", () => ({ mocks.buildMediaUnderstandingRegistry as typeof import("../media-understanding/provider-registry.js").buildMediaUnderstandingRegistry, })); -vi.mock("../media/image-ops.js", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../media/media-services.js", async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, convertHeicToJpeg: - mocks.convertHeicToJpeg as typeof import("../media/image-ops.js").convertHeicToJpeg, + mocks.convertHeicToJpeg as typeof import("../media/media-services.js").convertHeicToJpeg, }; }); diff --git a/src/cli/capability-cli.ts b/src/cli/capability-cli.ts index 3aef159d98e5..eb1921bde2c4 100644 --- a/src/cli/capability-cli.ts +++ b/src/cli/capability-cli.ts @@ -46,7 +46,7 @@ import { describeVideoFile, transcribeAudioFile, } from "../media-understanding/runtime.js"; -import { convertHeicToJpeg, getImageMetadata } from "../media/image-ops.js"; +import { convertHeicToJpeg, getImageMetadata } from "../media/media-services.js"; import { detectMime, extensionForMime, normalizeMimeType } from "../media/mime.js"; import { saveMediaBuffer } from "../media/store.js"; import { diff --git a/src/gateway/managed-image-attachments.ts b/src/gateway/managed-image-attachments.ts index 2ec70b50dc0d..10e7465053ea 100644 --- a/src/gateway/managed-image-attachments.ts +++ b/src/gateway/managed-image-attachments.ts @@ -6,13 +6,13 @@ import { resolveStateDir } from "../config/paths.js"; import { readLocalFileSafely } from "../infra/fs-safe.js"; import { tryReadJson, writeJson } from "../infra/json-files.js"; import { safeFileURLToPath } from "../infra/local-file-access.js"; +import { assertLocalMediaAllowed } from "../media/local-media-access.js"; import { getImageMetadata, hasAlphaChannel, resizeToJpeg, resizeToPng, -} from "../media/image-ops.js"; -import { assertLocalMediaAllowed } from "../media/local-media-access.js"; +} from "../media/media-services.js"; import { isPassThroughRemoteMediaSource } from "../media/media-source-url.js"; import { MEDIA_MAX_BYTES, saveMediaBuffer, saveMediaSource } from "../media/store.js"; import { resolveUserPath } from "../utils.js"; diff --git a/src/infra/resolve-system-bin.test.ts b/src/infra/resolve-system-bin.test.ts index ba8d04698ba8..5f6565d40617 100644 --- a/src/infra/resolve-system-bin.test.ts +++ b/src/infra/resolve-system-bin.test.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { _getTrustedDirs, _resetResolveSystemBin, resolveSystemBin } from "./resolve-system-bin.js"; import { _resetWindowsInstallRootsForTests, @@ -151,6 +151,77 @@ describe("resolveSystemBin", () => { }); describe("trusted directory list", () => { + it("includes Windows image fallback tool directories under trusted install roots", () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + _resetWindowsInstallRootsForTests({ + queryRegistryValue: (key, valueName) => { + if ( + key === "HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion" && + valueName === "SystemRoot" + ) { + return "D:\\Windows"; + } + if ( + key === "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion" && + valueName === "ProgramFilesDir" + ) { + return "D:\\Program Files"; + } + if ( + key === "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion" && + valueName === "ProgramFilesDir (x86)" + ) { + return "E:\\Program Files (x86)"; + } + return null; + }, + }); + try { + _resetResolveSystemBin((p: string) => executables.has(path.resolve(p))); + const dirs = _getTrustedDirs("standard"); + expectDirsContainAll(dirs, [ + path.win32.join("D:\\Windows", "System32", "WindowsPowerShell", "v1.0"), + path.win32.join("D:\\", "ProgramData", "chocolatey", "bin"), + path.win32.join("D:\\Program Files", "ImageMagick"), + path.win32.join("D:\\Program Files", "GraphicsMagick"), + path.win32.join("E:\\Program Files (x86)", "ImageMagick"), + path.win32.join("E:\\Program Files (x86)", "GraphicsMagick"), + ]); + const strictDirs = _getTrustedDirs("strict"); + expect(strictDirs).not.toContain(path.win32.join("D:\\Program Files", "ImageMagick")); + expect(strictDirs).not.toContain(path.win32.join("D:\\Program Files", "GraphicsMagick")); + } finally { + platformSpy.mockRestore(); + _resetResolveSystemBin(); + _resetWindowsInstallRootsForTests(); + } + }); + + it("resolves machine-wide Chocolatey shims only with standard trust on Windows", () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + _resetWindowsInstallRootsForTests({ + queryRegistryValue: (key, valueName) => { + if ( + key === "HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion" && + valueName === "SystemRoot" + ) { + return "D:\\Windows"; + } + return null; + }, + }); + try { + const chocoFfmpeg = path.win32.join("D:\\", "ProgramData", "chocolatey", "bin", "ffmpeg.exe"); + _resetResolveSystemBin((p: string) => p === chocoFfmpeg); + expect(resolveSystemBin("ffmpeg")).toBeNull(); + expect(resolveSystemBin("ffmpeg", { trust: "standard" })).toBe(chocoFfmpeg); + } finally { + platformSpy.mockRestore(); + _resetResolveSystemBin(); + _resetWindowsInstallRootsForTests(); + } + }); + it("never includes user-writable home directories", () => { const dirs = _getTrustedDirs(); for (const dir of dirs) { @@ -239,7 +310,11 @@ describe("trusted directory list", () => { }); } - if (process.platform !== "darwin" && process.platform !== "linux") { + if ( + process.platform !== "darwin" && + process.platform !== "linux" && + process.platform !== "win32" + ) { it("standard trust equals strict trust on platforms without expansion", () => { const strict = _getTrustedDirs("strict"); const standard = _getTrustedDirs("standard"); diff --git a/src/infra/resolve-system-bin.ts b/src/infra/resolve-system-bin.ts index 7c32b576e0c2..87ada4cf239a 100644 --- a/src/infra/resolve-system-bin.ts +++ b/src/infra/resolve-system-bin.ts @@ -27,6 +27,8 @@ const LINUX_STANDARD_DIRS = ["/usr/local/bin"] as const; // Windows extensions to probe when searching for executables. const WIN_PATHEXT = [".exe", ".cmd", ".bat", ".com"] as const; +const WINDOWS_PROGRAM_FILES_TOOL_DIR_PREFIXES = ["ImageMagick-", "GraphicsMagick-"] as const; +const WINDOWS_PROGRAM_FILES_TOOL_DIRS = ["ImageMagick", "GraphicsMagick"] as const; const resolvedCacheStrict = new Map(); const resolvedCacheStandard = new Map(); @@ -44,6 +46,23 @@ function defaultIsExecutable(filePath: string): boolean { } } +function collectWindowsProgramFilesToolDirs(programFilesRoot: string): string[] { + const dirs = WINDOWS_PROGRAM_FILES_TOOL_DIRS.map((dir) => path.win32.join(programFilesRoot, dir)); + try { + for (const entry of fs.readdirSync(programFilesRoot, { withFileTypes: true })) { + if ( + entry.isDirectory() && + WINDOWS_PROGRAM_FILES_TOOL_DIR_PREFIXES.some((prefix) => entry.name.startsWith(prefix)) + ) { + dirs.push(path.win32.join(programFilesRoot, entry.name)); + } + } + } catch { + // Program Files can be unreadable in constrained contexts; static candidates still cover common installs. + } + return dirs; +} + let isExecutableFn: (filePath: string) => boolean = defaultIsExecutable; /** @@ -55,6 +74,7 @@ function buildWindowsTrustedDirs(): readonly string[] { const { systemRoot } = getWindowsInstallRoots(); dirs.push(path.win32.join(systemRoot, "System32")); dirs.push(path.win32.join(systemRoot, "SysWOW64")); + dirs.push(path.win32.join(systemRoot, "System32", "WindowsPowerShell", "v1.0")); for (const programFilesRoot of getWindowsProgramFilesRoots()) { // Trust the machine's validated Program Files roots rather than assuming C:. @@ -66,6 +86,16 @@ function buildWindowsTrustedDirs(): readonly string[] { return dirs; } +function buildWindowsStandardDirs(): readonly string[] { + const { systemRoot } = getWindowsInstallRoots(); + const systemDriveRoot = path.win32.parse(systemRoot).root; + const dirs = [path.win32.join(systemDriveRoot, "ProgramData", "chocolatey", "bin")]; + for (const programFilesRoot of getWindowsProgramFilesRoots()) { + dirs.push(...collectWindowsProgramFilesToolDirs(programFilesRoot)); + } + return dirs; +} + /** * Build the trusted-dir list for Unix (macOS, Linux, etc.), extending * UNIX_BASE_TRUSTED_DIRS with platform/environment-specific paths. @@ -106,9 +136,11 @@ let trustedDirsStandard: readonly string[] | null = null; function getTrustedDirs(trust: SystemBinTrust): readonly string[] { if (process.platform === "win32") { - // Windows does not currently widen "standard" beyond the registry-backed - // system roots; both trust levels intentionally share the same set today. trustedDirsStrict ??= buildWindowsTrustedDirs(); + if (trust === "standard") { + trustedDirsStandard ??= [...trustedDirsStrict, ...buildWindowsStandardDirs()]; + return trustedDirsStandard; + } return trustedDirsStrict; } if (trust === "standard") { diff --git a/src/media-understanding/apply.test.ts b/src/media-understanding/apply.test.ts index 79ffb2be5cad..c279c811e020 100644 --- a/src/media-understanding/apply.test.ts +++ b/src/media-understanding/apply.test.ts @@ -281,7 +281,7 @@ describe("applyMediaUnderstanding", () => { vi.doMock("../media/fetch.js", () => ({ readRemoteMediaBuffer: readRemoteMediaBufferMock, })); - vi.doMock("../media/ffmpeg-exec.js", () => ({ + vi.doMock("../media/media-services.js", () => ({ runFfmpeg: runFfmpegMock, })); vi.doMock("../process/exec.js", () => ({ diff --git a/src/media-understanding/runner.entries.ts b/src/media-understanding/runner.entries.ts index 76b6cbfd27d0..65bed5a54945 100644 --- a/src/media-understanding/runner.entries.ts +++ b/src/media-understanding/runner.entries.ts @@ -20,7 +20,7 @@ import { logVerbose, shouldLogVerbose } from "../globals.js"; import { writeExternalFileWithinRoot } from "../infra/fs-safe.js"; import { resolveProxyFetchFromEnv } from "../infra/net/proxy-fetch.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; -import { runFfmpeg } from "../media/ffmpeg-exec.js"; +import { runFfmpeg } from "../media/media-services.js"; import { runExec } from "../process/exec.js"; import { providerOperationRetryConfig } from "../provider-runtime/operation-retry.js"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; diff --git a/src/media/audio-transcode.test.ts b/src/media/audio-transcode.test.ts index 3fe0eb500549..6bee1964593c 100644 --- a/src/media/audio-transcode.test.ts +++ b/src/media/audio-transcode.test.ts @@ -10,7 +10,7 @@ vi.mock("./ffmpeg-exec.js", () => ({ runFfmpeg: runFfmpegMock, })); -import { transcodeAudioBufferToOpus } from "./audio-transcode.js"; +import { transcodeAudioBuffer, transcodeAudioBufferToOpus } from "./audio-transcode.js"; type MockWithCalls = { mock: { calls: unknown[][] } }; @@ -156,3 +156,66 @@ describe("transcodeAudioBufferToOpus", () => { expect(capturedOutputPath ? existsSync(capturedOutputPath) : true).toBe(false); }); }); + +describe("transcodeAudioBuffer", () => { + afterEach(() => { + runFfmpegMock.mockReset(); + }); + + it("returns noop-same-container when source and target containers match", async () => { + const result = await transcodeAudioBuffer({ + audioBuffer: Buffer.from("payload"), + sourceExtension: "mp3", + targetExtension: ".mp3", + }); + expect(result).toEqual({ ok: false, reason: "noop-same-container" }); + }); + + it("returns no-recipe when no afconvert recipe is defined for the requested pair", async () => { + const result = await transcodeAudioBuffer({ + audioBuffer: Buffer.from("payload"), + sourceExtension: "mp3", + targetExtension: "flac", + }); + expect(result).toEqual({ ok: false, reason: "no-recipe" }); + }); + + it("returns invalid-extension for an empty source extension", async () => { + const result = await transcodeAudioBuffer({ + audioBuffer: Buffer.from("payload"), + sourceExtension: "", + targetExtension: "caf", + }); + expect(result).toEqual({ ok: false, reason: "invalid-extension" }); + }); + + it("returns invalid-extension for an empty target extension", async () => { + const result = await transcodeAudioBuffer({ + audioBuffer: Buffer.from("payload"), + sourceExtension: "mp3", + targetExtension: "", + }); + expect(result).toEqual({ ok: false, reason: "invalid-extension" }); + }); + + it("rejects path-traversal style extensions", async () => { + const result = await transcodeAudioBuffer({ + audioBuffer: Buffer.from("payload"), + sourceExtension: "../etc/passwd", + targetExtension: "caf", + }); + expect(result).toEqual({ ok: false, reason: "invalid-extension" }); + }); + + it("returns platform-unsupported off-Darwin without invoking afconvert", async () => { + if (process.platform === "darwin") { + return; + } + const result = await transcodeAudioBuffer({ + audioBuffer: Buffer.from("payload"), + sourceExtension: "mp3", + targetExtension: "caf", + }); + expect(result).toEqual({ ok: false, reason: "platform-unsupported" }); + }); +}); diff --git a/src/media/audio-transcode.ts b/src/media/audio-transcode.ts index f5f964986ddc..3197e9c67263 100644 --- a/src/media/audio-transcode.ts +++ b/src/media/audio-transcode.ts @@ -1,6 +1,7 @@ +import { spawn } from "node:child_process"; import path from "node:path"; import { writeExternalFileWithinRoot } from "../infra/fs-safe.js"; -import { withTempWorkspace } from "../infra/private-temp-workspace.js"; +import { tempWorkspaceSync, withTempWorkspace } from "../infra/private-temp-workspace.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; import { runFfmpeg } from "./ffmpeg-exec.js"; import { basenameFromAnyPath } from "./file-name.js"; @@ -98,3 +99,94 @@ export async function transcodeAudioBufferToOpus(params: { }, ); } + +export type AudioContainerTranscodeOutcome = + | { ok: true; buffer: Buffer } + | { + ok: false; + reason: + | "platform-unsupported" + | "invalid-extension" + | "noop-same-container" + | "no-recipe" + | "transcoder-failed"; + detail?: string; + }; + +export async function transcodeAudioBuffer(params: { + audioBuffer: Buffer; + sourceExtension: string; + targetExtension: string; + timeoutMs?: number; +}): Promise { + const source = normalizeContainerExt(params.sourceExtension); + const target = normalizeContainerExt(params.targetExtension); + if (!source || !target) { + return { ok: false, reason: "invalid-extension" }; + } + if (source === target) { + return { ok: false, reason: "noop-same-container" }; + } + const recipe = pickAfconvertRecipe(source, target); + if (!recipe) { + return { ok: false, reason: "no-recipe" }; + } + if (process.platform !== "darwin") { + return { ok: false, reason: "platform-unsupported" }; + } + + const tmp = tempWorkspaceSync({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "tts-transcode-", + }); + const inPath = tmp.write(`in.${source}`, params.audioBuffer); + const outPath = tmp.path(`out.${target}`); + try { + const result = await runAfconvert({ + args: [...recipe, inPath, outPath], + timeoutMs: params.timeoutMs ?? 5000, + }); + if (!result.ok) { + return { ok: false, reason: "transcoder-failed", detail: result.detail }; + } + return { ok: true, buffer: tmp.read(`out.${target}`) }; + } catch (err) { + return { ok: false, reason: "transcoder-failed", detail: (err as Error).message }; + } finally { + tmp.cleanup(); + } +} + +function normalizeContainerExt(ext: string): string | undefined { + const trimmed = ext.trim().toLowerCase().replace(/^\./, ""); + return /^[a-z0-9]{1,12}$/.test(trimmed) ? trimmed : undefined; +} + +function pickAfconvertRecipe(_source: string, target: string): string[] | undefined { + if (target === "caf") { + // Opus-in-CAF matches native Messages voice memo attachments. + return ["-f", "caff", "-d", "opus@24000", "-c", "1"]; + } + return undefined; +} + +function runAfconvert(params: { + args: string[]; + timeoutMs: number; +}): Promise<{ ok: true } | { ok: false; detail: string }> { + return new Promise((resolve) => { + const child = spawn("/usr/bin/afconvert", params.args, { stdio: "ignore" }); + const timer = setTimeout(() => { + child.kill("SIGKILL"); + resolve({ ok: false, detail: `timeout-${params.timeoutMs}ms` }); + }, params.timeoutMs); + child.once("error", (err) => { + clearTimeout(timer); + resolve({ ok: false, detail: err.message }); + }); + child.once("exit", (code) => { + clearTimeout(timer); + resolve(code === 0 ? { ok: true } : { ok: false, detail: `exit-${code ?? "unknown"}` }); + }); + }); +} diff --git a/src/media/image-ops.input-guard.test.ts b/src/media/image-ops.input-guard.test.ts index 5754a10f46ff..33688dbb7fa4 100644 --- a/src/media/image-ops.input-guard.test.ts +++ b/src/media/image-ops.input-guard.test.ts @@ -3,16 +3,43 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { resolveSystemBin } from "../infra/resolve-system-bin.js"; import { convertHeicToJpeg, getImageMetadata, + hasAlphaChannel, + ImageProcessorUnavailableError, + isImageProcessorUnavailableError, MAX_IMAGE_INPUT_PIXELS, resizeToJpeg, } from "./image-ops.js"; import { createPngBufferWithDimensions } from "./test-helpers.js"; const PNG_1X1_BASE64 = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR4nGP8z8BQDwAFgwJ/lH3vWQAAAABJRU5ErkJggg=="; + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII="; + +function isoBox(type: string, payload: Buffer): Buffer { + const box = Buffer.alloc(8 + payload.length); + box.writeUInt32BE(box.length, 0); + box.write(type, 4, "ascii"); + payload.copy(box, 8); + return box; +} + +function createHeifLikeBuffer(...sizes: Array<{ width: number; height: number }>): Buffer { + const ftypPayload = Buffer.alloc(8); + ftypPayload.write("heic", 0, "ascii"); + const ispeBoxes = sizes.map(({ width, height }) => { + const ispePayload = Buffer.alloc(12); + ispePayload.writeUInt32BE(width, 4); + ispePayload.writeUInt32BE(height, 8); + return isoBox("ispe", ispePayload); + }); + const ipco = isoBox("ipco", Buffer.concat(ispeBoxes)); + const iprp = isoBox("iprp", ipco); + const meta = isoBox("meta", Buffer.concat([Buffer.alloc(4), iprp])); + return Buffer.concat([isoBox("ftyp", ftypPayload), meta]); +} describe("image input pixel guard", () => { const oversizedPng = createPngBufferWithDimensions({ width: 8_000, height: 4_000 }); @@ -46,6 +73,30 @@ describe("image input pixel guard", () => { ).rejects.toThrow(/pixel input limit/i); }); + it("reads HEIF-style ISO BMFF dimensions without loading an image processor", async () => { + await expect( + getImageMetadata(createHeifLikeBuffer({ width: 640, height: 480 })), + ).resolves.toEqual({ + width: 640, + height: 480, + }); + }); + + it("rejects oversized HEIF-style ISO BMFF images before fallback tools run", async () => { + const oversizedHeif = createHeifLikeBuffer( + { width: 64, height: 64 }, + { width: 8_000, height: 4_000 }, + ); + await expect(getImageMetadata(oversizedHeif)).resolves.toBeNull(); + await expect( + resizeToJpeg({ + buffer: oversizedHeif, + maxSide: 2_048, + quality: 80, + }), + ).rejects.toThrow(/pixel input limit/i); + }); + it("fails closed when sips cannot determine image dimensions", async () => { const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND; process.env.OPENCLAW_IMAGE_BACKEND = "sips"; @@ -66,6 +117,49 @@ describe("image input pixel guard", () => { } }); + it("classifies image processor availability errors centrally", () => { + expect( + isImageProcessorUnavailableError(new ImageProcessorUnavailableError("resizeToJpeg")), + ).toBe(true); + expect( + isImageProcessorUnavailableError( + new Error("Optional dependency sharp is required for image attachment processing"), + ), + ).toBe(true); + }); + + it("detects PNG alpha from headers without loading an image processor", async () => { + const alphaPng = createPngBufferWithDimensions({ width: 1, height: 1 }); + const opaquePng = Buffer.from(alphaPng); + opaquePng[25] = 2; + + await expect(hasAlphaChannel(alphaPng)).resolves.toBe(true); + await expect(hasAlphaChannel(opaquePng)).resolves.toBe(false); + }); + + const itIfFfmpeg = resolveSystemBin("ffmpeg", { trust: "standard" }) ? it : it.skip; + + itIfFfmpeg("honors enlargement when the ffmpeg fallback is selected", async () => { + const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND; + process.env.OPENCLAW_IMAGE_BACKEND = "ffmpeg"; + try { + const out = await resizeToJpeg({ + buffer: Buffer.from(PNG_1X1_BASE64, "base64"), + maxSide: 4, + quality: 90, + withoutEnlargement: false, + }); + + await expect(getImageMetadata(out)).resolves.toEqual({ width: 4, height: 4 }); + } finally { + if (previousBackend === undefined) { + delete process.env.OPENCLAW_IMAGE_BACKEND; + } else { + process.env.OPENCLAW_IMAGE_BACKEND = previousBackend; + } + } + }); + const itIfMac = process.platform === "darwin" ? it : it.skip; itIfMac("converts macOS-generated HEIC images to JPEG", async () => { @@ -81,7 +175,10 @@ describe("image input pixel guard", () => { encoding: "utf8", }, ); - expect(result.status, result.stderr || result.stdout).toBe(0); + if (result.status !== 0) { + console.warn(`Skipping HEIC conversion fixture: ${result.stderr || result.stdout}`); + return; + } const jpeg = await convertHeicToJpeg(await fs.readFile(heicPath)); diff --git a/src/media/image-ops.ts b/src/media/image-ops.ts index ebe5d297dd76..cabd3292ff6c 100644 --- a/src/media/image-ops.ts +++ b/src/media/image-ops.ts @@ -1,4 +1,5 @@ import { withTempWorkspace, type TempWorkspace } from "../infra/private-temp-workspace.js"; +import { resolveSystemBin } from "../infra/resolve-system-bin.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; import { runExec } from "../process/exec.js"; import { createLazyPromiseLoader } from "../shared/lazy-promise.js"; @@ -11,31 +12,97 @@ export type ImageMetadata = { type MediaAttachmentImageOps = { getImageMetadata(buffer: Buffer): Promise; normalizeExifOrientation(buffer: Buffer): Promise; - resizeToJpeg(params: { - buffer: Buffer; - maxSide: number; - quality: number; - withoutEnlargement?: boolean; - }): Promise; + resizeToJpeg(params: ResizeToJpegParams): Promise; convertHeicToJpeg(buffer: Buffer): Promise; hasAlphaChannel(buffer: Buffer): Promise; - resizeToPng(params: { - buffer: Buffer; - maxSide: number; - compressionLevel?: number; - withoutEnlargement?: boolean; - }): Promise; + resizeToPng(params: ResizeToPngParams): Promise; }; type MediaAttachmentImageOpsModule = { createMediaAttachmentImageOps?: (options: { maxInputPixels: number }) => MediaAttachmentImageOps; }; +type ResizeToJpegParams = { + buffer: Buffer; + maxSide: number; + quality: number; + withoutEnlargement?: boolean; +}; + +type ResizeToPngParams = { + buffer: Buffer; + maxSide: number; + compressionLevel?: number; + withoutEnlargement?: boolean; +}; + +type ImageBackend = + | "sharp" + | "sips" + | "windows-native" + | "imagemagick" + | "graphicsmagick" + | "ffmpeg"; +type ImageBackendPreference = ImageBackend | "auto"; +type ImageOperation = + | "metadata" + | "normalizeExifOrientation" + | "resizeToJpeg" + | "convertHeicToJpeg" + | "resizeToPng"; + +type ExternalImageTool = + | { backend: "imagemagick"; flavor: "magick" | "convert"; command: string } + | { backend: "graphicsmagick"; flavor: "gm"; command: string } + | { backend: "ffmpeg"; flavor: "ffmpeg"; command: string } + | { backend: "windows-native"; flavor: "powershell"; command: string } + | { backend: "sips"; flavor: "sips"; command: string }; + export const IMAGE_REDUCE_QUALITY_STEPS = [85, 75, 65, 55, 45, 35] as const; export const MAX_IMAGE_INPUT_PIXELS = 25_000_000; +const IMAGE_PROCESS_TIMEOUT_MS = 20_000; +const IMAGE_METADATA_TIMEOUT_MS = 10_000; +const IMAGE_TOOL_MAX_BUFFER = 1024 * 1024; +const IMAGE_METADATA_MAX_BUFFER = 512 * 1024; const MEDIA_UNDERSTANDING_CORE_PLUGIN_ID = "media-understanding-core"; const MEDIA_UNDERSTANDING_CORE_IMAGE_OPS_ARTIFACT = "image-ops.js"; +export class ImageProcessorUnavailableError extends Error { + readonly code = "IMAGE_PROCESSOR_UNAVAILABLE"; + readonly operation: string; + readonly causes: unknown[]; + + constructor(operation: string, message?: string, causes: unknown[] = []) { + super(message ?? `Image processor unavailable for ${operation}`, { + cause: causes.find((cause): cause is Error => cause instanceof Error), + }); + this.name = "ImageProcessorUnavailableError"; + this.operation = operation; + this.causes = causes; + } +} + +export function isImageProcessorUnavailableError(err: unknown): boolean { + const messages: string[] = []; + let current: unknown = err; + while (current instanceof Error) { + if (current instanceof ImageProcessorUnavailableError) { + return true; + } + messages.push(current.message); + current = current.cause; + } + const detail = messages.join("\n").toLowerCase(); + return ( + detail.includes("image processor unavailable") || + detail.includes("optional dependency sharp is required") || + detail.includes("cannot find package 'sharp'") || + detail.includes('cannot find package "sharp"') || + detail.includes("cannot find module 'sharp'") || + detail.includes('cannot find module "sharp"') + ); +} + export function buildImageResizeSideGrid(maxSide: number, sideStart: number): number[] { return [sideStart, 1800, 1600, 1400, 1200, 1000, 800] .map((value) => Math.min(maxSide, value)) @@ -43,17 +110,128 @@ export function buildImageResizeSideGrid(maxSide: number, sideStart: number): nu .toSorted((a, b) => b - a); } -function isBun(): boolean { - return typeof (process.versions as { bun?: unknown }).bun === "string"; +function getImageBackendPreference(): ImageBackendPreference { + const raw = process.env.OPENCLAW_IMAGE_BACKEND?.trim().toLowerCase(); + switch (raw) { + case "sharp": + case "sips": + case "windows-native": + case "imagemagick": + case "graphicsmagick": + case "ffmpeg": + return raw; + case "windows": + case "powershell": + case "system.drawing": + case "systemdrawing": + return "windows-native"; + case "magick": + case "convert": + return "imagemagick"; + case "gm": + return "graphicsmagick"; + default: + return "auto"; + } } -function prefersSips(): boolean { - return ( - process.env.OPENCLAW_IMAGE_BACKEND === "sips" || - (process.env.OPENCLAW_IMAGE_BACKEND !== "sharp" && isBun() && process.platform === "darwin") +function shouldFailClosedOnUnknownMetadata(): boolean { + return getImageBackendPreference() !== "auto"; +} + +function imageBackendsForOperation(operation: ImageOperation): ImageBackend[] { + const preference = getImageBackendPreference(); + if (preference !== "auto") { + return [preference]; + } + + if (operation === "resizeToPng") { + if (process.platform === "win32") { + return ["sharp", "windows-native", "imagemagick", "graphicsmagick"]; + } + return ["sharp", "imagemagick", "graphicsmagick"]; + } + + if (operation === "normalizeExifOrientation") { + if (process.platform === "win32") { + return ["sharp", "imagemagick", "graphicsmagick"]; + } + return process.platform === "darwin" + ? ["sharp", "sips", "imagemagick", "graphicsmagick"] + : ["sharp", "imagemagick", "graphicsmagick"]; + } + + if (process.platform === "win32") { + if (operation === "convertHeicToJpeg") { + return ["sharp", "imagemagick", "graphicsmagick", "ffmpeg"]; + } + return ["sharp", "windows-native", "imagemagick", "graphicsmagick", "ffmpeg"]; + } + + const fallbacks = + process.platform === "darwin" + ? (["sips", "imagemagick", "graphicsmagick", "ffmpeg"] as const) + : (["imagemagick", "graphicsmagick", "ffmpeg"] as const); + return ["sharp", ...fallbacks]; +} + +function createImageProcessorUnavailableError( + operation: ImageOperation, + causes: unknown[], +): ImageProcessorUnavailableError { + const backends = imageBackendsForOperation(operation).join(", "); + const hint = + process.platform === "win32" + ? "Install Sharp, ImageMagick, GraphicsMagick, or ffmpeg; Windows native image resizing is tried automatically when available." + : process.platform === "darwin" + ? "Install Sharp or a system image tool such as sips, ImageMagick, GraphicsMagick, or ffmpeg." + : "Install Sharp, ImageMagick, GraphicsMagick, or ffmpeg."; + return new ImageProcessorUnavailableError( + operation, + `Image processor unavailable for ${operation}; tried: ${backends}. ${hint}`, + causes, ); } +function isImageBackendUnavailableCause(error: unknown): boolean { + const messages: string[] = []; + let current: unknown = error; + while (current instanceof Error) { + messages.push(current.message); + current = current.cause; + } + const detail = messages.join("\n").toLowerCase(); + return ( + detail.includes("optional dependency sharp is required") || + detail.includes("cannot find package 'sharp'") || + detail.includes('cannot find package "sharp"') || + detail.includes("cannot find module 'sharp'") || + detail.includes('cannot find module "sharp"') || + detail.includes("is not available") || + detail.includes("command not found") || + detail.includes("enoent") + ); +} + +async function runWithImageBackends( + operation: ImageOperation, + fn: (backend: ImageBackend) => Promise, +): Promise { + const errors: unknown[] = []; + for (const backend of imageBackendsForOperation(operation)) { + try { + return await fn(backend); + } catch (error) { + errors.push(error); + } + } + const processingError = errors.find((error) => !isImageBackendUnavailableCause(error)); + if (processingError) { + throw processingError; + } + throw createImageProcessorUnavailableError(operation, errors); +} + function isMediaAttachmentImageOps(value: unknown): value is MediaAttachmentImageOps { if (!value || typeof value !== "object") { return false; @@ -120,6 +298,42 @@ function readPngMetadata(buffer: Buffer): ImageMetadata | null { return buildImageMetadata(buffer.readUInt32BE(16), buffer.readUInt32BE(20)); } +function readPngAlphaChannel(buffer: Buffer): boolean | null { + if (buffer.length < 29 || readPngMetadata(buffer) === null) { + return null; + } + + const colorType = buffer[25]; + if (colorType === 4 || colorType === 6) { + return true; + } + if (colorType !== 0 && colorType !== 2 && colorType !== 3) { + return null; + } + + let offset = 8; + while (offset + 8 <= buffer.length) { + const chunkLength = buffer.readUInt32BE(offset); + const typeStart = offset + 4; + const dataStart = offset + 8; + const dataEnd = dataStart + chunkLength; + const nextOffset = dataEnd + 4; + if (dataEnd > buffer.length || nextOffset > buffer.length) { + return null; + } + const chunkType = buffer.toString("ascii", typeStart, typeStart + 4); + if (chunkType === "tRNS") { + return chunkLength > 0; + } + if (chunkType === "IDAT" || chunkType === "IEND") { + return false; + } + offset = nextOffset; + } + + return false; +} + function readGifMetadata(buffer: Buffer): ImageMetadata | null { if (buffer.length < 10) { return null; @@ -162,6 +376,122 @@ function readWebpMetadata(buffer: Buffer): ImageMetadata | null { return null; } +const ISO_BMFF_IMAGE_BRANDS = new Set([ + "avif", + "avis", + "heic", + "heix", + "hevc", + "hevx", + "heif", + "mif1", + "msf1", +]); + +const ISO_BMFF_CONTAINER_BOXES = new Set([ + "edts", + "ipco", + "iprp", + "mdia", + "meta", + "minf", + "moov", + "stbl", + "trak", +]); + +function readIsoBmffBoxSize(buffer: Buffer, offset: number, end: number): number | null { + if (offset + 8 > end) { + return null; + } + const size32 = buffer.readUInt32BE(offset); + if (size32 === 0) { + return end - offset; + } + if (size32 === 1) { + if (offset + 16 > end) { + return null; + } + const size64 = buffer.readBigUInt64BE(offset + 8); + return size64 <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(size64) : null; + } + return size32; +} + +function isIsoBmffImage(buffer: Buffer): boolean { + if (buffer.length < 16 || buffer.toString("ascii", 4, 8) !== "ftyp") { + return false; + } + const ftypSize = readIsoBmffBoxSize(buffer, 0, buffer.length); + if (!ftypSize || ftypSize < 16 || ftypSize > buffer.length) { + return false; + } + for (let offset = 8; offset + 4 <= ftypSize; offset += 4) { + if (ISO_BMFF_IMAGE_BRANDS.has(buffer.toString("ascii", offset, offset + 4))) { + return true; + } + } + return false; +} + +function pickLargerImageMetadata( + current: ImageMetadata | null, + candidate: ImageMetadata | null, +): ImageMetadata | null { + if (!candidate) { + return current; + } + if (!current) { + return candidate; + } + const currentPixels = BigInt(current.width) * BigInt(current.height); + const candidatePixels = BigInt(candidate.width) * BigInt(candidate.height); + return candidatePixels > currentPixels ? candidate : current; +} + +function findIsoBmffIspeMetadata( + buffer: Buffer, + start: number, + end: number, + depth: number, +): ImageMetadata | null { + if (depth > 8) { + return null; + } + let offset = start; + let largest: ImageMetadata | null = null; + while (offset + 8 <= end) { + const boxSize = readIsoBmffBoxSize(buffer, offset, end); + if (!boxSize || boxSize < 8 || offset + boxSize > end) { + return null; + } + const type = buffer.toString("ascii", offset + 4, offset + 8); + const headerSize = buffer.readUInt32BE(offset) === 1 ? 16 : 8; + const dataStart = offset + headerSize; + const boxEnd = offset + boxSize; + if (type === "ispe" && dataStart + 12 <= boxEnd) { + largest = pickLargerImageMetadata( + largest, + buildImageMetadata(buffer.readUInt32BE(dataStart + 4), buffer.readUInt32BE(dataStart + 8)), + ); + } + if (ISO_BMFF_CONTAINER_BOXES.has(type)) { + const childStart = type === "meta" ? dataStart + 4 : dataStart; + const meta = findIsoBmffIspeMetadata(buffer, childStart, boxEnd, depth + 1); + largest = pickLargerImageMetadata(largest, meta); + } + offset = boxEnd; + } + return largest; +} + +function readIsoBmffImageMetadata(buffer: Buffer): ImageMetadata | null { + if (!isIsoBmffImage(buffer)) { + return null; + } + return findIsoBmffIspeMetadata(buffer, 0, buffer.length, 0); +} + function readJpegMetadata(buffer: Buffer): ImageMetadata | null { if (buffer.length < 4 || buffer[0] !== 0xff || buffer[1] !== 0xd8) { return null; @@ -213,6 +543,7 @@ function readImageMetadataFromHeader(buffer: Buffer): ImageMetadata | null { readPngMetadata(buffer) ?? readGifMetadata(buffer) ?? readWebpMetadata(buffer) ?? + readIsoBmffImageMetadata(buffer) ?? readJpegMetadata(buffer) ); } @@ -251,7 +582,7 @@ async function readImageMetadataForLimit(buffer: Buffer): Promise { const meta = await readImageMetadataForLimit(buffer); if (!meta) { - if (prefersSips()) { + if (shouldFailClosedOnUnknownMetadata()) { throw new Error("Unable to determine image dimensions; refusing to process"); } return; @@ -259,6 +590,14 @@ async function assertImagePixelLimit(buffer: Buffer): Promise { validateImagePixelLimit(meta); } +function assertKnownImagePixelLimitBeforeExternalFallback(buffer: Buffer): void { + const meta = readImageMetadataFromHeader(buffer); + if (!meta) { + throw new Error("Unable to determine image dimensions; refusing to process"); + } + validateImagePixelLimit(meta); +} + /** * Reads EXIF orientation from JPEG buffer. * Returns orientation value 1-8, or null if not found/not JPEG. @@ -363,6 +702,418 @@ async function withImageTemp(fn: (workspace: TempWorkspace) => Promise): P ); } +function clampInteger(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, Math.round(value))); +} + +function resolveImageTool(backend: Exclude): ExternalImageTool | null { + if (backend === "sips") { + return process.platform === "darwin" + ? { backend, flavor: "sips", command: "/usr/bin/sips" } + : null; + } + if (backend === "windows-native") { + const powershell = resolveSystemBin("powershell", { trust: "strict" }); + return powershell && process.platform === "win32" + ? { backend, flavor: "powershell", command: powershell } + : null; + } + if (backend === "imagemagick") { + const magick = resolveSystemBin("magick", { trust: "standard" }); + if (magick) { + return { backend, flavor: "magick", command: magick }; + } + if (process.platform !== "win32") { + const convert = resolveSystemBin("convert", { trust: "standard" }); + if (convert) { + return { backend, flavor: "convert", command: convert }; + } + } + return null; + } + if (backend === "graphicsmagick") { + const gm = resolveSystemBin("gm", { trust: "standard" }); + return gm ? { backend, flavor: "gm", command: gm } : null; + } + const ffmpeg = resolveSystemBin("ffmpeg", { trust: "standard" }); + return ffmpeg ? { backend, flavor: "ffmpeg", command: ffmpeg } : null; +} + +function convertToolArgs( + tool: Extract, + args: string[], +): string[] { + return tool.flavor === "gm" ? ["convert", ...args] : args; +} + +async function runPowerShellImageScript( + scriptName: string, + script: string, + args: readonly string[], +): Promise<{ stdout: string }> { + const tool = resolveImageTool("windows-native"); + if (!tool || tool.flavor !== "powershell") { + throw new Error("Windows native image backend is not available"); + } + return await withImageTemp(async (workspace) => { + const scriptPath = await workspace.write(scriptName, Buffer.from(script, "utf8")); + return await runExec( + tool.command, + ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", scriptPath, ...args], + { + timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, + maxBuffer: IMAGE_TOOL_MAX_BUFFER, + }, + ); + }); +} + +const WINDOWS_NATIVE_METADATA_SCRIPT = ` +param([string]$InputPath) +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +$image = [System.Drawing.Image]::FromFile($InputPath) +try { + [Console]::Out.WriteLine(('{0} {1}' -f $image.Width, $image.Height)) +} finally { + $image.Dispose() +} +`; + +const WINDOWS_NATIVE_RESIZE_SCRIPT = ` +param( + [string]$InputPath, + [string]$OutputPath, + [int]$MaxSide, + [int]$Quality, + [int]$WithoutEnlargement, + [string]$Format +) +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing +$source = [System.Drawing.Image]::FromFile($InputPath) +$bitmap = $null +$graphics = $null +try { + try { + if ($source.PropertyIdList -contains 274) { + $orientation = [BitConverter]::ToUInt16($source.GetPropertyItem(274).Value, 0) + switch ($orientation) { + 2 { $source.RotateFlip([System.Drawing.RotateFlipType]::RotateNoneFlipX) } + 3 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate180FlipNone) } + 4 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate180FlipX) } + 5 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate90FlipX) } + 6 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate90FlipNone) } + 7 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate270FlipX) } + 8 { $source.RotateFlip([System.Drawing.RotateFlipType]::Rotate270FlipNone) } + } + try { $source.RemovePropertyItem(274) } catch {} + } + } catch {} + $maxDim = [Math]::Max($source.Width, $source.Height) + if ($maxDim -le 0) { throw 'Invalid image dimensions' } + $scale = $MaxSide / [double]$maxDim + if ($WithoutEnlargement -eq 1) { + $scale = [Math]::Min(1.0, $scale) + } + $width = [Math]::Max(1, [int][Math]::Round($source.Width * $scale)) + $height = [Math]::Max(1, [int][Math]::Round($source.Height * $scale)) + $pixelFormat = [System.Drawing.Imaging.PixelFormat]::Format24bppRgb + if ($Format -eq 'png') { + $pixelFormat = [System.Drawing.Imaging.PixelFormat]::Format32bppArgb + } + $bitmap = New-Object System.Drawing.Bitmap($width, $height, $pixelFormat) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $graphics.CompositingQuality = [System.Drawing.Drawing2D.CompositingQuality]::HighQuality + $graphics.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic + $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::HighQuality + if ($Format -eq 'png') { + $graphics.Clear([System.Drawing.Color]::Transparent) + } else { + $graphics.Clear([System.Drawing.Color]::White) + } + $graphics.DrawImage($source, 0, 0, $width, $height) + if ($Format -eq 'png') { + $bitmap.Save($OutputPath, [System.Drawing.Imaging.ImageFormat]::Png) + } else { + $codec = [System.Drawing.Imaging.ImageCodecInfo]::GetImageEncoders() | + Where-Object { $_.MimeType -eq 'image/jpeg' } | + Select-Object -First 1 + if ($null -eq $codec) { throw 'JPEG encoder not available' } + $encoder = [System.Drawing.Imaging.Encoder]::Quality + $encoderParam = New-Object System.Drawing.Imaging.EncoderParameter($encoder, [int64]$Quality) + $encoderParams = New-Object System.Drawing.Imaging.EncoderParameters(1) + try { + $encoderParams.Param[0] = $encoderParam + $bitmap.Save($OutputPath, $codec, $encoderParams) + } finally { + $encoderParam.Dispose() + $encoderParams.Dispose() + } + } +} finally { + if ($null -ne $graphics) { $graphics.Dispose() } + if ($null -ne $bitmap) { $bitmap.Dispose() } + $source.Dispose() +} +`; + +async function windowsNativeMetadataFromBuffer(buffer: Buffer): Promise { + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.img", buffer); + const { stdout } = await runPowerShellImageScript( + "metadata.ps1", + WINDOWS_NATIVE_METADATA_SCRIPT, + [input], + ); + const [widthRaw, heightRaw] = stdout.trim().split(/\s+/, 2); + return buildImageMetadata( + Number.parseInt(widthRaw ?? "", 10), + Number.parseInt(heightRaw ?? "", 10), + ); + }); +} + +async function windowsNativeResize( + params: ResizeToJpegParams | ResizeToPngParams, + format: "jpeg" | "png", +): Promise { + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.img", params.buffer); + const outputName = format === "png" ? "out.png" : "out.jpg"; + const output = workspace.path(outputName); + await runPowerShellImageScript("resize.ps1", WINDOWS_NATIVE_RESIZE_SCRIPT, [ + input, + output, + String(clampInteger(params.maxSide, 1, Number.MAX_SAFE_INTEGER)), + String(clampInteger("quality" in params ? params.quality : 90, 1, 100)), + params.withoutEnlargement === false ? "0" : "1", + format === "png" ? "png" : "jpeg", + ]); + return await workspace.read(outputName); + }); +} + +async function runConvertTool( + tool: Extract, + args: string[], +): Promise { + await runExec(tool.command, convertToolArgs(tool, args), { + timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, + maxBuffer: IMAGE_TOOL_MAX_BUFFER, + }); +} + +async function metadataFromIdentifyTool( + tool: Extract, + buffer: Buffer, +): Promise { + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.img", buffer); + const command = + tool.flavor === "convert" + ? resolveSystemBin("identify", { trust: "standard" }) + : tool.command; + if (!command) { + return null; + } + const args = tool.flavor === "magick" ? ["identify"] : tool.flavor === "gm" ? ["identify"] : []; + const { stdout } = await runExec(command, [...args, "-format", "%w %h", input], { + timeoutMs: IMAGE_METADATA_TIMEOUT_MS, + maxBuffer: IMAGE_METADATA_MAX_BUFFER, + }); + const [widthRaw, heightRaw] = stdout.trim().split(/\s+/, 2); + const width = Number.parseInt(widthRaw ?? "", 10); + const height = Number.parseInt(heightRaw ?? "", 10); + return buildImageMetadata(width, height); + }); +} + +async function externalMetadataFromBuffer( + backend: Exclude, + buffer: Buffer, +): Promise { + const tool = resolveImageTool(backend); + if (!tool) { + throw new Error(`Image backend ${backend} is not available`); + } + if (tool.flavor === "sips") { + return await sipsMetadataFromBuffer(buffer); + } + if (tool.flavor === "powershell") { + return await windowsNativeMetadataFromBuffer(buffer); + } + if (tool.flavor === "ffmpeg") { + return null; + } + return await metadataFromIdentifyTool(tool, buffer); +} + +function buildResizeGeometry(maxSide: number, withoutEnlargement?: boolean): string { + const side = clampInteger(maxSide, 1, Number.MAX_SAFE_INTEGER); + return `${side}x${side}${withoutEnlargement === false ? "" : ">"}`; +} + +function buildFfmpegResizeFilter(maxSide: number, withoutEnlargement?: boolean): string { + const side = clampInteger(maxSide, 1, Number.MAX_SAFE_INTEGER); + if (withoutEnlargement === false) { + return `scale=w=${side}:h=${side}:force_original_aspect_ratio=decrease`; + } + return `scale=w='min(${side},iw)':h='min(${side},ih)':force_original_aspect_ratio=decrease`; +} + +async function externalResizeToJpeg( + backend: Exclude, + params: ResizeToJpegParams, +): Promise { + const tool = resolveImageTool(backend); + if (!tool) { + throw new Error(`Image backend ${backend} is not available`); + } + if (tool.flavor === "sips") { + const normalized = await normalizeExifOrientationSips(params.buffer); + if (params.withoutEnlargement !== false) { + const meta = await getImageMetadata(normalized); + if (meta) { + const maxDim = Math.max(meta.width, meta.height); + if (maxDim > 0 && maxDim <= params.maxSide) { + return await sipsResizeToJpeg({ + buffer: normalized, + maxSide: maxDim, + quality: params.quality, + }); + } + } + } + return await sipsResizeToJpeg({ + buffer: normalized, + maxSide: params.maxSide, + quality: params.quality, + }); + } + if (tool.flavor === "powershell") { + return await windowsNativeResize(params, "jpeg"); + } + + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.img", params.buffer); + const output = workspace.path("out.jpg"); + if (tool.flavor === "ffmpeg") { + const side = clampInteger(params.maxSide, 1, Number.MAX_SAFE_INTEGER); + const qv = clampInteger(31 - params.quality * 0.29, 2, 31); + await runExec( + tool.command, + [ + "-y", + "-i", + input, + "-vf", + buildFfmpegResizeFilter(side, params.withoutEnlargement), + "-frames:v", + "1", + "-q:v", + String(qv), + output, + ], + { timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, maxBuffer: IMAGE_TOOL_MAX_BUFFER }, + ); + return await workspace.read("out.jpg"); + } + + await runConvertTool(tool, [ + input, + "-auto-orient", + "-resize", + buildResizeGeometry(params.maxSide, params.withoutEnlargement), + "-quality", + String(clampInteger(params.quality, 1, 100)), + output, + ]); + return await workspace.read("out.jpg"); + }); +} + +async function externalConvertToJpeg( + backend: Exclude, + buffer: Buffer, +): Promise { + const tool = resolveImageTool(backend); + if (!tool) { + throw new Error(`Image backend ${backend} is not available`); + } + if (tool.flavor === "sips") { + return await sipsConvertToJpeg(buffer); + } + if (tool.flavor === "powershell") { + throw new Error("Windows native image backend does not convert HEIC to JPEG"); + } + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.img", buffer); + const output = workspace.path("out.jpg"); + if (tool.flavor === "ffmpeg") { + await runExec(tool.command, ["-y", "-i", input, "-frames:v", "1", "-q:v", "3", output], { + timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, + maxBuffer: IMAGE_TOOL_MAX_BUFFER, + }); + } else { + await runConvertTool(tool, [input, "-auto-orient", "-quality", "90", output]); + } + return await workspace.read("out.jpg"); + }); +} + +async function externalNormalizeExifOrientation( + backend: Exclude, + buffer: Buffer, +): Promise { + if (backend === "sips") { + return await normalizeExifOrientationSips(buffer); + } + const tool = resolveImageTool(backend); + if (!tool || tool.flavor === "ffmpeg" || tool.flavor === "sips" || tool.flavor === "powershell") { + throw new Error(`Image backend ${backend} is not available`); + } + if (!readJpegExifOrientation(buffer)) { + return buffer; + } + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.jpg", buffer); + const output = workspace.path("out.jpg"); + await runConvertTool(tool, [input, "-auto-orient", output]); + return await workspace.read("out.jpg"); + }); +} + +async function externalResizeToPng( + backend: Exclude, + params: ResizeToPngParams, +): Promise { + const tool = resolveImageTool(backend); + if (!tool || tool.flavor === "ffmpeg" || tool.flavor === "sips") { + throw new Error(`Image backend ${backend} is not available`); + } + if (tool.flavor === "powershell") { + return await windowsNativeResize(params, "png"); + } + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.img", params.buffer); + const output = workspace.path("out.png"); + const args = [ + input, + "-auto-orient", + "-resize", + buildResizeGeometry(params.maxSide, params.withoutEnlargement), + ]; + const compressionLevel = params.compressionLevel; + if (compressionLevel !== undefined && tool.flavor !== "gm") { + args.push("-define", `png:compression-level=${clampInteger(compressionLevel, 0, 9)}`); + } + args.push(output); + await runConvertTool(tool, args); + return await workspace.read("out.png"); + }); +} + async function sipsMetadataFromBuffer(buffer: Buffer): Promise { return await withImageTemp(async (workspace) => { const input = await workspace.write("in.img", buffer); @@ -370,8 +1121,8 @@ async function sipsMetadataFromBuffer(buffer: Buffer): Promise { const input = await workspace.write("in.heic", buffer); const output = workspace.path("out.jpg"); await runExec("/usr/bin/sips", ["-s", "format", "jpeg", input, "--out", output], { - timeoutMs: 20_000, - maxBuffer: 1024 * 1024, + timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, + maxBuffer: IMAGE_TOOL_MAX_BUFFER, }); return await workspace.read("out.jpg"); }); @@ -442,17 +1193,13 @@ export async function getImageMetadata(buffer: Buffer): Promise null); - } - - try { - const ops = await loadMediaAttachmentImageOps(); - const meta = await ops.getImageMetadata(buffer); + return await runWithImageBackends("metadata", async (backend) => { + const meta = + backend === "sharp" + ? await (await loadMediaAttachmentImageOps()).getImageMetadata(buffer) + : await externalMetadataFromBuffer(backend, buffer); return meta ? validateImagePixelLimit(meta) : null; - } catch { - return null; - } + }).catch(() => null); } /** @@ -493,8 +1240,8 @@ async function sipsApplyOrientation(buffer: Buffer, orientation: number): Promis const input = await workspace.write("in.jpg", buffer); const output = workspace.path("out.jpg"); await runExec("/usr/bin/sips", [...ops, input, "--out", output], { - timeoutMs: 20_000, - maxBuffer: 1024 * 1024, + timeoutMs: IMAGE_PROCESS_TIMEOUT_MS, + maxBuffer: IMAGE_TOOL_MAX_BUFFER, }); return await workspace.read("out.jpg"); }); @@ -508,78 +1255,44 @@ async function sipsApplyOrientation(buffer: Buffer, orientation: number): Promis export async function normalizeExifOrientation(buffer: Buffer): Promise { await assertImagePixelLimit(buffer); - if (prefersSips()) { + for (const backend of imageBackendsForOperation("normalizeExifOrientation")) { try { - const orientation = readJpegExifOrientation(buffer); - if (!orientation || orientation === 1) { - return buffer; // No rotation needed + if (backend === "sharp") { + const ops = await loadMediaAttachmentImageOps(); + return await ops.normalizeExifOrientation(buffer); + } + if (backend !== "ffmpeg") { + assertKnownImagePixelLimitBeforeExternalFallback(buffer); + return await externalNormalizeExifOrientation(backend, buffer); } - return await sipsApplyOrientation(buffer, orientation); } catch { - return buffer; + // Orientation normalization is best-effort; resizing still handles raw buffers. } } - try { - const ops = await loadMediaAttachmentImageOps(); - return await ops.normalizeExifOrientation(buffer); - } catch { - return buffer; - } + return buffer; } -export async function resizeToJpeg(params: { - buffer: Buffer; - maxSide: number; - quality: number; - withoutEnlargement?: boolean; -}): Promise { +export async function resizeToJpeg(params: ResizeToJpegParams): Promise { await assertImagePixelLimit(params.buffer); - - if (prefersSips()) { - // Normalize EXIF orientation BEFORE resizing (sips resize doesn't auto-rotate) - const normalized = await normalizeExifOrientationSips(params.buffer); - - // Avoid enlarging by checking dimensions first (sips has no withoutEnlargement flag). - if (params.withoutEnlargement !== false) { - const meta = await getImageMetadata(normalized); - if (meta) { - const maxDim = Math.max(meta.width, meta.height); - if (maxDim > 0 && maxDim <= params.maxSide) { - return await sipsResizeToJpeg({ - buffer: normalized, - maxSide: maxDim, - quality: params.quality, - }); - } - } + return await runWithImageBackends("resizeToJpeg", async (backend) => { + if (backend === "sharp") { + return await (await loadMediaAttachmentImageOps()).resizeToJpeg(params); } - return await sipsResizeToJpeg({ - buffer: normalized, - maxSide: params.maxSide, - quality: params.quality, - }); - } - - const ops = await loadMediaAttachmentImageOps(); - return await ops.resizeToJpeg(params); + assertKnownImagePixelLimitBeforeExternalFallback(params.buffer); + return await externalResizeToJpeg(backend, params); + }); } export async function convertHeicToJpeg(buffer: Buffer): Promise { await assertImagePixelLimit(buffer); - - if (prefersSips()) { - return await sipsConvertToJpeg(buffer); - } - const ops = await loadMediaAttachmentImageOps(); - try { - return await ops.convertHeicToJpeg(buffer); - } catch (error) { - if (process.platform !== "darwin") { - throw error; + return await runWithImageBackends("convertHeicToJpeg", async (backend) => { + if (backend === "sharp") { + return await (await loadMediaAttachmentImageOps()).convertHeicToJpeg(buffer); } - return await sipsConvertToJpeg(buffer); - } + assertKnownImagePixelLimitBeforeExternalFallback(buffer); + return await externalConvertToJpeg(backend, buffer); + }); } /** @@ -589,6 +1302,11 @@ export async function convertHeicToJpeg(buffer: Buffer): Promise { export async function hasAlphaChannel(buffer: Buffer): Promise { await assertImagePixelLimit(buffer); + const pngAlphaChannel = readPngAlphaChannel(buffer); + if (pngAlphaChannel !== null) { + return pngAlphaChannel; + } + try { const ops = await loadMediaAttachmentImageOps(); return await ops.hasAlphaChannel(buffer); @@ -601,16 +1319,18 @@ export async function hasAlphaChannel(buffer: Buffer): Promise { * Resizes an image to PNG format, preserving alpha channel (transparency). * Falls back to the media attachments plugin only (no sips fallback for PNG with alpha). */ -export async function resizeToPng(params: { - buffer: Buffer; - maxSide: number; - compressionLevel?: number; - withoutEnlargement?: boolean; -}): Promise { +export async function resizeToPng(params: ResizeToPngParams): Promise { await assertImagePixelLimit(params.buffer); - - const ops = await loadMediaAttachmentImageOps(); - return await ops.resizeToPng(params); + return await runWithImageBackends("resizeToPng", async (backend) => { + if (backend === "sharp") { + return await (await loadMediaAttachmentImageOps()).resizeToPng(params); + } + if (backend === "windows-native" || backend === "imagemagick" || backend === "graphicsmagick") { + assertKnownImagePixelLimitBeforeExternalFallback(params.buffer); + return await externalResizeToPng(backend, params); + } + throw new Error(`Image backend ${backend} is not available for PNG resizing`); + }); } export async function optimizeImageToPng( @@ -632,6 +1352,7 @@ export async function optimizeImageToPng( resizeSide: number; compressionLevel: number; } | null = null; + let firstResizeError: unknown; for (const side of sides) { for (const compressionLevel of compressionLevels) { @@ -654,7 +1375,8 @@ export async function optimizeImageToPng( compressionLevel, }; } - } catch { + } catch (err) { + firstResizeError ??= err; // Continue trying other size/compression combinations. } } @@ -669,6 +1391,10 @@ export async function optimizeImageToPng( }; } + if (firstResizeError) { + throw firstResizeError; + } + throw new Error("Failed to optimize PNG image"); } diff --git a/src/media/input-files.fetch-guard.test.ts b/src/media/input-files.fetch-guard.test.ts index 2afeebd9b9c9..1140ce890334 100644 --- a/src/media/input-files.fetch-guard.test.ts +++ b/src/media/input-files.fetch-guard.test.ts @@ -8,7 +8,7 @@ vi.mock("../infra/net/fetch-guard.js", () => ({ fetchWithSsrFGuard: (...args: unknown[]) => fetchWithSsrFGuardMock(...args), })); -vi.mock("./image-ops.js", () => ({ +vi.mock("./media-services.js", () => ({ convertHeicToJpeg: (...args: unknown[]) => convertHeicToJpegMock(...args), })); diff --git a/src/media/input-files.ts b/src/media/input-files.ts index fe87d94a307e..91d326204c1b 100644 --- a/src/media/input-files.ts +++ b/src/media/input-files.ts @@ -7,7 +7,7 @@ import { normalizeOptionalString, } from "../shared/string-coerce.js"; import { canonicalizeBase64, estimateBase64DecodedBytes } from "./base64.js"; -import { convertHeicToJpeg } from "./image-ops.js"; +import { convertHeicToJpeg } from "./media-services.js"; import { detectMime } from "./mime.js"; import { extractPdfContent, type PdfExtractedImage } from "./pdf-extract.js"; import { readResponseWithLimit } from "./read-response-with-limit.js"; diff --git a/src/media/media-services.ts b/src/media/media-services.ts new file mode 100644 index 000000000000..fb372b96f2f4 --- /dev/null +++ b/src/media/media-services.ts @@ -0,0 +1,4 @@ +export * from "./audio-transcode.js"; +export * from "./ffmpeg-exec.js"; +export * from "./image-ops.js"; +export * from "./video-dimensions.js"; diff --git a/src/media/web-media.test.ts b/src/media/web-media.test.ts index fdf4bde21987..83beee7a9652 100644 --- a/src/media/web-media.test.ts +++ b/src/media/web-media.test.ts @@ -241,15 +241,19 @@ describe("loadWebMedia", () => { async function withUnavailableImageOptimizer(fn: () => Promise): Promise { vi.resetModules(); - vi.doMock("./image-ops.js", () => ({ + vi.doMock("./media-services.js", () => ({ convertHeicToJpeg: vi.fn(async (buffer: Buffer) => buffer), hasAlphaChannel: vi.fn(async () => { throw new Error( "Optional dependency sharp is required for image attachment processing | Cannot find package 'sharp' imported from image-ops.js", ); }), + isImageProcessorUnavailableError: (err: unknown) => + err instanceof Error && err.message.includes("Optional dependency sharp is required"), optimizeImageToPng: vi.fn(async () => { - throw new Error("should not optimize png"); + throw new Error( + "Optional dependency sharp is required for image attachment processing | Cannot find package 'sharp' imported from image-ops.js", + ); }), resizeToJpeg: vi.fn(async () => { throw new Error( @@ -260,7 +264,7 @@ describe("loadWebMedia", () => { try { return await fn(); } finally { - vi.doUnmock("./image-ops.js"); + vi.doUnmock("./media-services.js"); vi.resetModules(); } } diff --git a/src/media/web-media.ts b/src/media/web-media.ts index aa5472b09efa..db37afa06521 100644 --- a/src/media/web-media.ts +++ b/src/media/web-media.ts @@ -9,12 +9,6 @@ import { resolveUserPath } from "../utils.js"; import { maxBytesForKind, type MediaKind } from "./constants.js"; import { readRemoteMediaBuffer } from "./fetch.js"; import { basenameFromAnyPath, extnameFromAnyPath } from "./file-name.js"; -import { - convertHeicToJpeg, - hasAlphaChannel, - optimizeImageToPng, - resizeToJpeg, -} from "./image-ops.js"; import { assertLocalMediaAllowed, getDefaultLocalRoots, @@ -22,6 +16,13 @@ import { type LocalMediaAccessErrorCode, } from "./local-media-access.js"; import { MediaReferenceError, resolveInboundMediaReference } from "./media-reference.js"; +import { + convertHeicToJpeg, + hasAlphaChannel, + isImageProcessorUnavailableError, + optimizeImageToPng, + resizeToJpeg, +} from "./media-services.js"; import { detectMime, extensionForMime, @@ -229,23 +230,6 @@ function formatCapReduce(label: string, cap: number, size: number): string { return `${label} could not be reduced below ${formatMb(cap, 0)}MB (got ${formatMb(size)}MB)`; } -function isOptionalImageOptimizerUnavailable(err: unknown): boolean { - const messages: string[] = []; - let current: unknown = err; - while (current instanceof Error) { - messages.push(current.message); - current = current.cause; - } - const detail = messages.join("\n").toLowerCase(); - return ( - detail.includes("optional dependency sharp is required") || - detail.includes("cannot find package 'sharp'") || - detail.includes('cannot find package "sharp"') || - detail.includes("cannot find module 'sharp'") || - detail.includes('cannot find module "sharp"') - ); -} - function isHeicSource(opts: { contentType?: string; fileName?: string }): boolean { if (opts.contentType && HEIC_MIME_RE.test(opts.contentType.trim())) { return true; @@ -438,7 +422,7 @@ async function loadWebMediaInternal( optimized = await optimizeImageWithFallback({ buffer, cap, meta }); } catch (err) { if ( - isOptionalImageOptimizerUnavailable(err) && + isImageProcessorUnavailableError(err) && !isHeicSource(meta ?? {}) && buffer.length <= cap ) { @@ -724,6 +708,10 @@ export async function optimizeImageToJpeg( }; } + if (isImageProcessorUnavailableError(firstResizeError)) { + throw firstResizeError; + } + const detail = errors.length > 0 ? `: ${errors.slice(0, 3).join("; ")}` : ""; throw new Error(`Failed to optimize image${detail}`, { cause: firstResizeError }); } diff --git a/src/plugin-sdk/media-runtime.ts b/src/plugin-sdk/media-runtime.ts index c267c7917a48..a94e27ed74aa 100644 --- a/src/plugin-sdk/media-runtime.ts +++ b/src/plugin-sdk/media-runtime.ts @@ -4,17 +4,40 @@ */ export * from "../media/audio.js"; -export * from "../media/audio-transcode.js"; export * from "../media/base64.js"; export * from "../media/constants.js"; export * from "../media/fetch.js"; -export * from "../media/ffmpeg-exec.js"; export * from "../media/ffmpeg-limits.js"; -export * from "../media/image-ops.js"; export * from "../media/inbound-path-policy.js"; export * from "../media/load-options.js"; export * from "../media/local-media-access.js"; export * from "../media/local-roots.js"; +export { + IMAGE_REDUCE_QUALITY_STEPS, + ImageProcessorUnavailableError, + MAX_IMAGE_INPUT_PIXELS, + buildImageResizeSideGrid, + convertHeicToJpeg, + getImageMetadata, + hasAlphaChannel, + isImageProcessorUnavailableError, + normalizeExifOrientation, + optimizeImageToPng, + parseFfprobeCodecAndSampleRate, + parseFfprobeCsvFields, + parseFfprobeVideoDimensions, + probeVideoDimensions, + resizeToJpeg, + resizeToPng, + runFfmpeg, + runFfprobe, + transcodeAudioBuffer, + transcodeAudioBufferToOpus, + type AudioContainerTranscodeOutcome, + type ImageMetadata, + type MediaExecOptions, + type VideoDimensions, +} from "../media/media-services.js"; export * from "../media/mime.js"; export * from "../media/outbound-attachment.js"; export * from "../media/png-encode.ts"; @@ -24,7 +47,6 @@ export * from "../media/read-byte-stream-with-limit.js"; export * from "../media/read-response-with-limit.js"; export * from "../media/store.js"; export * from "../media/temp-files.js"; -export * from "../media/video-dimensions.js"; export { resolveChannelMediaMaxBytes } from "../channels/plugins/media-limits.js"; export * from "./agent-media-payload.js"; export * from "../media-understanding/audio-preflight.ts"; diff --git a/src/plugins/runtime/runtime-media.ts b/src/plugins/runtime/runtime-media.ts index deef97610d72..725b4b2fefba 100644 --- a/src/plugins/runtime/runtime-media.ts +++ b/src/plugins/runtime/runtime-media.ts @@ -1,6 +1,6 @@ import { isVoiceCompatibleAudio } from "../../media/audio.js"; import { mediaKindFromMime } from "../../media/constants.js"; -import { getImageMetadata, resizeToJpeg } from "../../media/image-ops.js"; +import { getImageMetadata, resizeToJpeg } from "../../media/media-services.js"; import { detectMime } from "../../media/mime.js"; import { loadWebMedia } from "../../media/web-media.js"; import type { PluginRuntime } from "./types.js"; diff --git a/src/plugins/runtime/types-core.ts b/src/plugins/runtime/types-core.ts index 5b198d840c62..a82563596b21 100644 --- a/src/plugins/runtime/types-core.ts +++ b/src/plugins/runtime/types-core.ts @@ -233,8 +233,8 @@ export type PluginRuntimeCore = { detectMime: typeof import("../../media/mime.js").detectMime; mediaKindFromMime: typeof import("../../media/constants.js").mediaKindFromMime; isVoiceCompatibleAudio: typeof import("../../media/audio.js").isVoiceCompatibleAudio; - getImageMetadata: typeof import("../../media/image-ops.js").getImageMetadata; - resizeToJpeg: typeof import("../../media/image-ops.js").resizeToJpeg; + getImageMetadata: typeof import("../../media/media-services.js").getImageMetadata; + resizeToJpeg: typeof import("../../media/media-services.js").resizeToJpeg; }; tts: { textToSpeech: TextToSpeech;