mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 21:07:01 -06:00
Merge remote-tracking branch 'origin/main' into fix/bundled-channel-load-doctor-hint
# Conflicts: # CHANGELOG.md
This commit is contained in:
@@ -39,6 +39,7 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
### Fixes
|
||||
|
||||
- Media: install Sharp with the root package and fall back to sips, Windows native imaging, ImageMagick, GraphicsMagick, or ffmpeg for image resizing/conversion when Sharp is unavailable. Fixes #83401. Thanks @scotthuang.
|
||||
- Channels/bundled: append `openclaw doctor --fix` guidance to the bundled-channel load warnings emitted on `ERR_MODULE_NOT_FOUND` / `MODULE_NOT_FOUND` (including those wrapped on `.cause` by the native-require loader), so users hitting unstaged plugin runtime deps (e.g. `nostr-tools`) see an actionable repair hint instead of a bare module-not-found warning. (#76974) Thanks @BSG2000.
|
||||
- Telegram: deliver generated media completions back into forum topics by preserving topic IDs across requester-agent handoff. (#83556) Thanks @fuller-stack-dev.
|
||||
- Gateway: defer update-check startup until after readiness so package update checks no longer block sidecar-ready startup, while preserving update broadcasts and shutdown cleanup. (#83520) Thanks @samzong.
|
||||
|
||||
@@ -3,6 +3,22 @@ import { describe, expect, it } from "vitest";
|
||||
import { normalizeBrowserScreenshot } from "./screenshot.js";
|
||||
|
||||
describe("browser screenshot normalization", () => {
|
||||
const unavailableImageBackend = process.platform === "win32" ? "sips" : "windows-native";
|
||||
|
||||
async function withUnavailableImageBackend<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND;
|
||||
process.env.OPENCLAW_IMAGE_BACKEND = unavailableImageBackend;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
if (previousBackend === undefined) {
|
||||
delete process.env.OPENCLAW_IMAGE_BACKEND;
|
||||
} else {
|
||||
process.env.OPENCLAW_IMAGE_BACKEND = previousBackend;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
it("shrinks oversized images to <=2000x2000 and <=5MB", async () => {
|
||||
const bigPng = await sharp({
|
||||
create: {
|
||||
@@ -47,4 +63,27 @@ describe("browser screenshot normalization", () => {
|
||||
|
||||
expect(normalized.buffer.equals(jpeg)).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects screenshots above max side when no image processor is available", async () => {
|
||||
const png = await sharp({
|
||||
create: {
|
||||
width: 420,
|
||||
height: 120,
|
||||
channels: 3,
|
||||
background: { r: 12, g: 34, b: 56 },
|
||||
},
|
||||
})
|
||||
.png({ compressionLevel: 9 })
|
||||
.toBuffer();
|
||||
expect(png.byteLength).toBeLessThan(5 * 1024 * 1024);
|
||||
|
||||
await withUnavailableImageBackend(async () => {
|
||||
await expect(
|
||||
normalizeBrowserScreenshot(png, {
|
||||
maxSide: 120,
|
||||
maxBytes: 5 * 1024 * 1024,
|
||||
}),
|
||||
).rejects.toThrow(/image processor unavailable/i);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,8 +2,9 @@ import {
|
||||
buildImageResizeSideGrid,
|
||||
getImageMetadata,
|
||||
IMAGE_REDUCE_QUALITY_STEPS,
|
||||
isImageProcessorUnavailableError,
|
||||
resizeToJpeg,
|
||||
} from "../media/image-ops.js";
|
||||
} from "../media/media-services.js";
|
||||
|
||||
export const DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE = 2000;
|
||||
export const DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES = 5 * 1024 * 1024;
|
||||
@@ -31,15 +32,25 @@ export async function normalizeBrowserScreenshot(
|
||||
const sideGrid = buildImageResizeSideGrid(maxSide, sideStart);
|
||||
|
||||
let smallest: { buffer: Buffer; size: number } | null = null;
|
||||
let processorUnavailableError: unknown;
|
||||
|
||||
for (const side of sideGrid) {
|
||||
for (const quality of IMAGE_REDUCE_QUALITY_STEPS) {
|
||||
const out = await resizeToJpeg({
|
||||
buffer,
|
||||
maxSide: side,
|
||||
quality,
|
||||
withoutEnlargement: true,
|
||||
});
|
||||
let out: Buffer;
|
||||
try {
|
||||
out = await resizeToJpeg({
|
||||
buffer,
|
||||
maxSide: side,
|
||||
quality,
|
||||
withoutEnlargement: true,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isImageProcessorUnavailableError(err)) {
|
||||
processorUnavailableError = err;
|
||||
break;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (!smallest || out.byteLength < smallest.size) {
|
||||
smallest = { buffer: out, size: out.byteLength };
|
||||
@@ -49,6 +60,13 @@ export async function normalizeBrowserScreenshot(
|
||||
return { buffer: out, contentType: "image/jpeg" };
|
||||
}
|
||||
}
|
||||
if (processorUnavailableError) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (processorUnavailableError) {
|
||||
throw processorUnavailableError;
|
||||
}
|
||||
|
||||
const best = smallest?.buffer ?? buffer;
|
||||
|
||||
@@ -3,4 +3,4 @@ export {
|
||||
buildImageResizeSideGrid,
|
||||
getImageMetadata,
|
||||
resizeToJpeg,
|
||||
} from "../sdk-setup-tools.js";
|
||||
} from "./media-services.js";
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
IMAGE_REDUCE_QUALITY_STEPS,
|
||||
buildImageResizeSideGrid,
|
||||
getImageMetadata,
|
||||
isImageProcessorUnavailableError,
|
||||
resizeToJpeg,
|
||||
} from "../sdk-setup-tools.js";
|
||||
@@ -23,6 +23,7 @@ export {
|
||||
IMAGE_REDUCE_QUALITY_STEPS,
|
||||
buildImageResizeSideGrid,
|
||||
getImageMetadata,
|
||||
isImageProcessorUnavailableError,
|
||||
resizeToJpeg,
|
||||
} from "openclaw/plugin-sdk/media-runtime";
|
||||
export { detectMime } from "openclaw/plugin-sdk/media-mime";
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { isInboundPathAllowed } from "openclaw/plugin-sdk/media-runtime";
|
||||
import { saveMediaBuffer } from "openclaw/plugin-sdk/media-store";
|
||||
import { buildRandomTempFilePath } from "openclaw/plugin-sdk/temp-path";
|
||||
import { loadWebMedia } from "openclaw/plugin-sdk/web-media";
|
||||
import type { IMessageAttachment } from "./types.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const HEIC_CONVERSION_TIMEOUT_MS = 15_000;
|
||||
const HEIC_CONVERSION_MAX_BUFFER_BYTES = 64 * 1024;
|
||||
|
||||
export type StagedIMessageAttachment = {
|
||||
path: string;
|
||||
contentType?: string;
|
||||
@@ -73,43 +66,6 @@ async function resolveAllowedCanonicalAttachmentPath(params: {
|
||||
return canonicalPath;
|
||||
}
|
||||
|
||||
async function convertHeicToJpegWithSips(sourcePath: string, maxBytes: number): Promise<Buffer> {
|
||||
const tempPath = buildRandomTempFilePath({
|
||||
prefix: "openclaw-imessage",
|
||||
extension: "jpg",
|
||||
});
|
||||
try {
|
||||
await execFileAsync(
|
||||
"sips",
|
||||
[
|
||||
"-s",
|
||||
"format",
|
||||
"jpeg",
|
||||
"-s",
|
||||
"formatOptions",
|
||||
"90",
|
||||
"-Z",
|
||||
"4096",
|
||||
sourcePath,
|
||||
"--out",
|
||||
tempPath,
|
||||
],
|
||||
{
|
||||
timeout: HEIC_CONVERSION_TIMEOUT_MS,
|
||||
maxBuffer: HEIC_CONVERSION_MAX_BUFFER_BYTES,
|
||||
killSignal: "SIGKILL",
|
||||
},
|
||||
);
|
||||
const stat = await fs.stat(tempPath);
|
||||
if (stat.size > maxBytes) {
|
||||
throw new Error(`converted media exceeds ${Math.round(maxBytes / (1024 * 1024))}MB limit`);
|
||||
}
|
||||
return await fs.readFile(tempPath);
|
||||
} finally {
|
||||
await fs.rm(tempPath, { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function readAttachmentBuffer(params: {
|
||||
attachmentPath: string;
|
||||
mimeType?: string | null;
|
||||
@@ -142,11 +98,20 @@ async function readAttachmentBuffer(params: {
|
||||
|
||||
if (isHeicAttachment(params.attachmentPath, params.mimeType)) {
|
||||
try {
|
||||
const convert = params.deps.convertHeicToJpeg ?? convertHeicToJpegWithSips;
|
||||
const convert = params.deps.convertHeicToJpeg;
|
||||
const converted = convert
|
||||
? {
|
||||
buffer: await convert(canonicalPath, params.maxBytes),
|
||||
fileName: jpegFilenameForAttachment(params.attachmentPath),
|
||||
}
|
||||
: await loadWebMedia(canonicalPath, {
|
||||
maxBytes: params.maxBytes,
|
||||
localRoots: [path.dirname(canonicalPath)],
|
||||
});
|
||||
return {
|
||||
buffer: await convert(canonicalPath, params.maxBytes),
|
||||
buffer: converted.buffer,
|
||||
contentType: "image/jpeg",
|
||||
originalFilename: jpegFilenameForAttachment(params.attachmentPath),
|
||||
originalFilename: converted.fileName ?? jpegFilenameForAttachment(params.attachmentPath),
|
||||
};
|
||||
} catch (err) {
|
||||
params.deps.logVerbose?.(
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { transcodeAudioBuffer } from "./audio-transcode.js";
|
||||
|
||||
describe("transcodeAudioBuffer", () => {
|
||||
it("returns noop-same-container when source and target containers match", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "mp3",
|
||||
targetExtension: ".mp3",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "noop-same-container" });
|
||||
});
|
||||
|
||||
it("returns no-recipe when no afconvert recipe is defined for the requested pair", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "mp3",
|
||||
targetExtension: "flac",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "no-recipe" });
|
||||
});
|
||||
|
||||
it("returns invalid-extension for an empty source extension", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "",
|
||||
targetExtension: "caf",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "invalid-extension" });
|
||||
});
|
||||
|
||||
it("returns invalid-extension for an empty target extension", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "mp3",
|
||||
targetExtension: "",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "invalid-extension" });
|
||||
});
|
||||
|
||||
it("rejects path-traversal style extensions", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "../etc/passwd",
|
||||
targetExtension: "caf",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "invalid-extension" });
|
||||
});
|
||||
|
||||
it("returns platform-unsupported off-Darwin without invoking afconvert", async () => {
|
||||
if (process.platform === "darwin") {
|
||||
// macOS: a valid mp3→caf request would proceed to spawn `afconvert`,
|
||||
// which we don't want to run from a unit test. The Darwin happy path
|
||||
// is exercised end-to-end via the native voice-memo flow.
|
||||
return;
|
||||
}
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "mp3",
|
||||
targetExtension: "caf",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "platform-unsupported" });
|
||||
});
|
||||
});
|
||||
@@ -1,122 +0,0 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { tempWorkspaceSync, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox";
|
||||
|
||||
type TranscodeOutcome =
|
||||
| { ok: true; buffer: Buffer }
|
||||
| {
|
||||
ok: false;
|
||||
reason:
|
||||
| "platform-unsupported"
|
||||
| "invalid-extension"
|
||||
| "noop-same-container"
|
||||
| "no-recipe"
|
||||
| "transcoder-failed";
|
||||
detail?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Best-effort audio container transcode using macOS `afconvert`.
|
||||
*
|
||||
* Used by the TTS pipeline to pre-encode synthesized audio into a channel's
|
||||
* preferred container (see `ChannelTtsVoiceDeliveryCapabilities.preferAudioFileFormat`)
|
||||
* so the channel's downstream does not have to perform a container
|
||||
* conversion of its own. Returns a discriminated outcome so callers can
|
||||
* distinguish "we didn't try" (platform/recipe/noop) from "we tried and the
|
||||
* transcoder failed", which is the case worth logging.
|
||||
*
|
||||
* Currently only macOS is supported because `afconvert` is the only widely
|
||||
* available encoder we ship a recipe for.
|
||||
*/
|
||||
export async function transcodeAudioBuffer(params: {
|
||||
audioBuffer: Buffer;
|
||||
sourceExtension: string;
|
||||
targetExtension: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<TranscodeOutcome> {
|
||||
// Validate inputs first so callers get a specific reason regardless of
|
||||
// host platform. Platform-unsupported is the gate immediately before the
|
||||
// actual `afconvert` invocation.
|
||||
const source = normalizeExt(params.sourceExtension);
|
||||
const target = normalizeExt(params.targetExtension);
|
||||
if (!source || !target) {
|
||||
return { ok: false, reason: "invalid-extension" };
|
||||
}
|
||||
if (source === target) {
|
||||
return { ok: false, reason: "noop-same-container" };
|
||||
}
|
||||
const recipe = pickAfconvertRecipe(source, target);
|
||||
if (!recipe) {
|
||||
return { ok: false, reason: "no-recipe" };
|
||||
}
|
||||
if (process.platform !== "darwin") {
|
||||
return { ok: false, reason: "platform-unsupported" };
|
||||
}
|
||||
|
||||
const tmp = tempWorkspaceSync({
|
||||
rootDir: resolvePreferredOpenClawTmpDir(),
|
||||
prefix: "tts-transcode-",
|
||||
});
|
||||
const inPath = tmp.write(`in.${source}`, params.audioBuffer);
|
||||
const outPath = tmp.path(`out.${target}`);
|
||||
try {
|
||||
const result = await runAfconvert({
|
||||
args: [...recipe, inPath, outPath],
|
||||
timeoutMs: params.timeoutMs ?? 5000,
|
||||
});
|
||||
if (!result.ok) {
|
||||
return { ok: false, reason: "transcoder-failed", detail: result.detail };
|
||||
}
|
||||
return { ok: true, buffer: tmp.read(`out.${target}`) };
|
||||
} catch (err) {
|
||||
return { ok: false, reason: "transcoder-failed", detail: (err as Error).message };
|
||||
} finally {
|
||||
tmp.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeExt(ext: string): string | undefined {
|
||||
// Pattern matches the sibling helper in src/media/audio-transcode.ts: a short
|
||||
// alphanumeric extension token. Keeps the value safe to interpolate into
|
||||
// tmp-file names below without introducing a path-traversal surface.
|
||||
const trimmed = ext.trim().toLowerCase().replace(/^\./, "");
|
||||
return /^[a-z0-9]{1,12}$/.test(trimmed) ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function pickAfconvertRecipe(_source: string, target: string): string[] | undefined {
|
||||
// Currently only the MP3->CAF path used by native Messages voice memos.
|
||||
if (target === "caf") {
|
||||
// Opus-in-CAF, mono, 24 kHz. Validated against macOS 15.x Messages.app's
|
||||
// native voice-memo CAF descriptor (1 ch, 24000 Hz, opus); other CAF
|
||||
// flavors (PCM, AAC) get downgraded to plain audio attachments along the
|
||||
// Messages.app path. If iMessage stops rendering the result
|
||||
// as a voice memo after a system update, try forcing frames-per-packet
|
||||
// explicitly via `opus@24000#480` and re-validate. See #72506.
|
||||
return ["-f", "caff", "-d", "opus@24000", "-c", "1"];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function runAfconvert(params: {
|
||||
args: string[];
|
||||
timeoutMs: number;
|
||||
}): Promise<{ ok: true } | { ok: false; detail: string }> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("/usr/bin/afconvert", params.args, { stdio: "ignore" });
|
||||
const timer = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
resolve({ ok: false, detail: `timeout-${params.timeoutMs}ms` });
|
||||
}, params.timeoutMs);
|
||||
child.once("error", (err) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: false, detail: err.message });
|
||||
});
|
||||
child.once("exit", (code) => {
|
||||
clearTimeout(timer);
|
||||
if (code === 0) {
|
||||
resolve({ ok: true });
|
||||
} else {
|
||||
resolve({ ok: false, detail: `exit-${code ?? "unknown"}` });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -55,7 +55,7 @@ const transcodeAudioBufferMock = vi.hoisted(() =>
|
||||
>(async () => ({ ok: false, reason: "platform-unsupported" })),
|
||||
);
|
||||
|
||||
vi.mock("./audio-transcode.js", () => ({
|
||||
vi.mock("openclaw/plugin-sdk/media-runtime", () => ({
|
||||
transcodeAudioBuffer: transcodeAudioBufferMock,
|
||||
}));
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
} from "openclaw/plugin-sdk/config-contracts";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core";
|
||||
import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime";
|
||||
import {
|
||||
markReplyPayloadAsTtsSupplement,
|
||||
resolveSendableOutboundReplyParts,
|
||||
@@ -50,7 +51,6 @@ import {
|
||||
type TtsDirectiveParseResult,
|
||||
type TtsConfigResolutionContext,
|
||||
} from "../api.js";
|
||||
import { transcodeAudioBuffer } from "./audio-transcode.js";
|
||||
|
||||
export type {
|
||||
ResolvedTtsConfig,
|
||||
|
||||
@@ -1837,6 +1837,7 @@
|
||||
"vitest": "4.1.6"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"sharp": "0.34.5",
|
||||
"sqlite-vec": "0.1.9"
|
||||
},
|
||||
"overrides": {
|
||||
|
||||
Generated
+3
@@ -262,6 +262,9 @@ importers:
|
||||
specifier: 4.1.6
|
||||
version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@25.8.0)(@vitest/browser-playwright@4.1.6)(@vitest/coverage-v8@4.1.6)(jsdom@29.1.1(@noble/hashes@2.0.1))(vite@8.0.13(@types/node@25.8.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.0)(yaml@2.9.0))
|
||||
optionalDependencies:
|
||||
sharp:
|
||||
specifier: 0.34.5
|
||||
version: 0.34.5
|
||||
sqlite-vec:
|
||||
specifier: 0.1.9
|
||||
version: 0.1.9
|
||||
|
||||
@@ -10,7 +10,7 @@ const { callGateway } = vi.hoisted(() => ({
|
||||
}));
|
||||
|
||||
vi.mock("../gateway/call.js", () => ({ callGateway }));
|
||||
vi.mock("../media/image-ops.js", () => ({
|
||||
vi.mock("../media/media-services.js", () => ({
|
||||
getImageMetadata: vi.fn(async () => ({ width: 1, height: 1 })),
|
||||
resizeToJpeg: vi.fn(async () => Buffer.from("jpeg")),
|
||||
}));
|
||||
|
||||
@@ -3,6 +3,22 @@ import { describe, expect, it } from "vitest";
|
||||
import { sanitizeContentBlocksImages, sanitizeImageBlocks } from "./tool-images.js";
|
||||
|
||||
describe("tool image sanitizing", () => {
|
||||
const unavailableImageBackend = process.platform === "win32" ? "sips" : "windows-native";
|
||||
|
||||
async function withUnavailableImageBackend<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND;
|
||||
process.env.OPENCLAW_IMAGE_BACKEND = unavailableImageBackend;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
if (previousBackend === undefined) {
|
||||
delete process.env.OPENCLAW_IMAGE_BACKEND;
|
||||
} else {
|
||||
process.env.OPENCLAW_IMAGE_BACKEND = previousBackend;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const getImageBlock = (
|
||||
blocks: Awaited<ReturnType<typeof sanitizeContentBlocksImages>>,
|
||||
): (typeof blocks)[number] & { type: "image"; data: string; mimeType?: string } => {
|
||||
@@ -86,6 +102,29 @@ describe("tool image sanitizing", () => {
|
||||
expect(image.mimeType).toBe("image/jpeg");
|
||||
}, 20_000);
|
||||
|
||||
it("drops images above max dimension when no image processor is available", async () => {
|
||||
const png = await createWidePng();
|
||||
expect(png.byteLength).toBeLessThan(5 * 1024 * 1024);
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: "image" as const,
|
||||
data: png.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
},
|
||||
];
|
||||
|
||||
const out = await withUnavailableImageBackend(() =>
|
||||
sanitizeContentBlocksImages(blocks, "test", { maxDimensionPx: 120 }),
|
||||
);
|
||||
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0].type).toBe("text");
|
||||
if (out[0].type === "text") {
|
||||
expect(out[0].text).toMatch(/image processor unavailable/i);
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("corrects mismatched jpeg mimeType", async () => {
|
||||
const jpeg = await sharp({
|
||||
create: {
|
||||
|
||||
@@ -6,8 +6,9 @@ import {
|
||||
buildImageResizeSideGrid,
|
||||
getImageMetadata,
|
||||
IMAGE_REDUCE_QUALITY_STEPS,
|
||||
isImageProcessorUnavailableError,
|
||||
resizeToJpeg,
|
||||
} from "../media/image-ops.js";
|
||||
} from "../media/media-services.js";
|
||||
import {
|
||||
DEFAULT_IMAGE_MAX_BYTES,
|
||||
DEFAULT_IMAGE_MAX_DIMENSION_PX,
|
||||
@@ -187,14 +188,24 @@ async function resizeImageBase64IfNeeded(params: {
|
||||
const sideGrid = buildImageResizeSideGrid(params.maxDimensionPx, sideStart);
|
||||
|
||||
let smallest: { buffer: Buffer; size: number } | null = null;
|
||||
let processorUnavailableError: unknown;
|
||||
for (const side of sideGrid) {
|
||||
for (const quality of IMAGE_REDUCE_QUALITY_STEPS) {
|
||||
const out = await resizeToJpeg({
|
||||
buffer: buf,
|
||||
maxSide: side,
|
||||
quality,
|
||||
withoutEnlargement: true,
|
||||
});
|
||||
let out: Buffer;
|
||||
try {
|
||||
out = await resizeToJpeg({
|
||||
buffer: buf,
|
||||
maxSide: side,
|
||||
quality,
|
||||
withoutEnlargement: true,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isImageProcessorUnavailableError(err)) {
|
||||
processorUnavailableError = err;
|
||||
break;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!smallest || out.byteLength < smallest.size) {
|
||||
smallest = { buffer: out, size: out.byteLength };
|
||||
}
|
||||
@@ -239,6 +250,13 @@ async function resizeImageBase64IfNeeded(params: {
|
||||
};
|
||||
}
|
||||
}
|
||||
if (processorUnavailableError) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (processorUnavailableError) {
|
||||
throw processorUnavailableError;
|
||||
}
|
||||
|
||||
const best = smallest?.buffer ?? buf;
|
||||
|
||||
@@ -10,7 +10,7 @@ const taskRuntimeMocks = vi.hoisted(() => ({
|
||||
vi.mock("../../tasks/detached-task-runtime.js", () => taskRuntimeMocks);
|
||||
|
||||
let imageGenerationRuntime: typeof import("../../image-generation/runtime.js");
|
||||
let imageOps: typeof import("../../media/image-ops.js");
|
||||
let imageOps: typeof import("../../media/media-services.js");
|
||||
let splitMediaFromOutput: typeof import("../../media/parse.js").splitMediaFromOutput;
|
||||
let mediaStore: typeof import("../../media/store.js");
|
||||
let webMedia: typeof import("../../media/web-media.js");
|
||||
@@ -288,7 +288,7 @@ describe("createImageGenerateTool", () => {
|
||||
};
|
||||
});
|
||||
imageGenerationRuntime = await import("../../image-generation/runtime.js");
|
||||
imageOps = await import("../../media/image-ops.js");
|
||||
imageOps = await import("../../media/media-services.js");
|
||||
({ splitMediaFromOutput } = await import("../../media/parse.js"));
|
||||
mediaStore = await import("../../media/store.js");
|
||||
webMedia = await import("../../media/web-media.js");
|
||||
|
||||
@@ -25,11 +25,11 @@ import {
|
||||
resolveConfiguredMediaMaxBytes,
|
||||
resolveGeneratedMediaMaxBytes,
|
||||
} from "../../media/configured-max-bytes.js";
|
||||
import { getImageMetadata } from "../../media/image-ops.js";
|
||||
import {
|
||||
classifyMediaReferenceSource,
|
||||
normalizeMediaReferenceSource,
|
||||
} from "../../media/media-reference.js";
|
||||
import { getImageMetadata } from "../../media/media-services.js";
|
||||
import { saveMediaBuffer } from "../../media/store.js";
|
||||
import { loadWebMedia } from "../../media/web-media.js";
|
||||
import { resolveUserPath } from "../../utils.js";
|
||||
|
||||
@@ -242,12 +242,12 @@ vi.mock("../media-understanding/provider-registry.js", () => ({
|
||||
mocks.buildMediaUnderstandingRegistry as typeof import("../media-understanding/provider-registry.js").buildMediaUnderstandingRegistry,
|
||||
}));
|
||||
|
||||
vi.mock("../media/image-ops.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../media/image-ops.js")>();
|
||||
vi.mock("../media/media-services.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../media/media-services.js")>();
|
||||
return {
|
||||
...actual,
|
||||
convertHeicToJpeg:
|
||||
mocks.convertHeicToJpeg as typeof import("../media/image-ops.js").convertHeicToJpeg,
|
||||
mocks.convertHeicToJpeg as typeof import("../media/media-services.js").convertHeicToJpeg,
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ import {
|
||||
describeVideoFile,
|
||||
transcribeAudioFile,
|
||||
} from "../media-understanding/runtime.js";
|
||||
import { convertHeicToJpeg, getImageMetadata } from "../media/image-ops.js";
|
||||
import { convertHeicToJpeg, getImageMetadata } from "../media/media-services.js";
|
||||
import { detectMime, extensionForMime, normalizeMimeType } from "../media/mime.js";
|
||||
import { saveMediaBuffer } from "../media/store.js";
|
||||
import {
|
||||
|
||||
@@ -6,13 +6,13 @@ import { resolveStateDir } from "../config/paths.js";
|
||||
import { readLocalFileSafely } from "../infra/fs-safe.js";
|
||||
import { tryReadJson, writeJson } from "../infra/json-files.js";
|
||||
import { safeFileURLToPath } from "../infra/local-file-access.js";
|
||||
import { assertLocalMediaAllowed } from "../media/local-media-access.js";
|
||||
import {
|
||||
getImageMetadata,
|
||||
hasAlphaChannel,
|
||||
resizeToJpeg,
|
||||
resizeToPng,
|
||||
} from "../media/image-ops.js";
|
||||
import { assertLocalMediaAllowed } from "../media/local-media-access.js";
|
||||
} from "../media/media-services.js";
|
||||
import { isPassThroughRemoteMediaSource } from "../media/media-source-url.js";
|
||||
import { MEDIA_MAX_BYTES, saveMediaBuffer, saveMediaSource } from "../media/store.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { _getTrustedDirs, _resetResolveSystemBin, resolveSystemBin } from "./resolve-system-bin.js";
|
||||
import {
|
||||
_resetWindowsInstallRootsForTests,
|
||||
@@ -151,6 +151,77 @@ describe("resolveSystemBin", () => {
|
||||
});
|
||||
|
||||
describe("trusted directory list", () => {
|
||||
it("includes Windows image fallback tool directories under trusted install roots", () => {
|
||||
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32");
|
||||
_resetWindowsInstallRootsForTests({
|
||||
queryRegistryValue: (key, valueName) => {
|
||||
if (
|
||||
key === "HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion" &&
|
||||
valueName === "SystemRoot"
|
||||
) {
|
||||
return "D:\\Windows";
|
||||
}
|
||||
if (
|
||||
key === "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion" &&
|
||||
valueName === "ProgramFilesDir"
|
||||
) {
|
||||
return "D:\\Program Files";
|
||||
}
|
||||
if (
|
||||
key === "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion" &&
|
||||
valueName === "ProgramFilesDir (x86)"
|
||||
) {
|
||||
return "E:\\Program Files (x86)";
|
||||
}
|
||||
return null;
|
||||
},
|
||||
});
|
||||
try {
|
||||
_resetResolveSystemBin((p: string) => executables.has(path.resolve(p)));
|
||||
const dirs = _getTrustedDirs("standard");
|
||||
expectDirsContainAll(dirs, [
|
||||
path.win32.join("D:\\Windows", "System32", "WindowsPowerShell", "v1.0"),
|
||||
path.win32.join("D:\\", "ProgramData", "chocolatey", "bin"),
|
||||
path.win32.join("D:\\Program Files", "ImageMagick"),
|
||||
path.win32.join("D:\\Program Files", "GraphicsMagick"),
|
||||
path.win32.join("E:\\Program Files (x86)", "ImageMagick"),
|
||||
path.win32.join("E:\\Program Files (x86)", "GraphicsMagick"),
|
||||
]);
|
||||
const strictDirs = _getTrustedDirs("strict");
|
||||
expect(strictDirs).not.toContain(path.win32.join("D:\\Program Files", "ImageMagick"));
|
||||
expect(strictDirs).not.toContain(path.win32.join("D:\\Program Files", "GraphicsMagick"));
|
||||
} finally {
|
||||
platformSpy.mockRestore();
|
||||
_resetResolveSystemBin();
|
||||
_resetWindowsInstallRootsForTests();
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves machine-wide Chocolatey shims only with standard trust on Windows", () => {
|
||||
const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32");
|
||||
_resetWindowsInstallRootsForTests({
|
||||
queryRegistryValue: (key, valueName) => {
|
||||
if (
|
||||
key === "HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion" &&
|
||||
valueName === "SystemRoot"
|
||||
) {
|
||||
return "D:\\Windows";
|
||||
}
|
||||
return null;
|
||||
},
|
||||
});
|
||||
try {
|
||||
const chocoFfmpeg = path.win32.join("D:\\", "ProgramData", "chocolatey", "bin", "ffmpeg.exe");
|
||||
_resetResolveSystemBin((p: string) => p === chocoFfmpeg);
|
||||
expect(resolveSystemBin("ffmpeg")).toBeNull();
|
||||
expect(resolveSystemBin("ffmpeg", { trust: "standard" })).toBe(chocoFfmpeg);
|
||||
} finally {
|
||||
platformSpy.mockRestore();
|
||||
_resetResolveSystemBin();
|
||||
_resetWindowsInstallRootsForTests();
|
||||
}
|
||||
});
|
||||
|
||||
it("never includes user-writable home directories", () => {
|
||||
const dirs = _getTrustedDirs();
|
||||
for (const dir of dirs) {
|
||||
@@ -239,7 +310,11 @@ describe("trusted directory list", () => {
|
||||
});
|
||||
}
|
||||
|
||||
if (process.platform !== "darwin" && process.platform !== "linux") {
|
||||
if (
|
||||
process.platform !== "darwin" &&
|
||||
process.platform !== "linux" &&
|
||||
process.platform !== "win32"
|
||||
) {
|
||||
it("standard trust equals strict trust on platforms without expansion", () => {
|
||||
const strict = _getTrustedDirs("strict");
|
||||
const standard = _getTrustedDirs("standard");
|
||||
|
||||
@@ -27,6 +27,8 @@ const LINUX_STANDARD_DIRS = ["/usr/local/bin"] as const;
|
||||
|
||||
// Windows extensions to probe when searching for executables.
|
||||
const WIN_PATHEXT = [".exe", ".cmd", ".bat", ".com"] as const;
|
||||
const WINDOWS_PROGRAM_FILES_TOOL_DIR_PREFIXES = ["ImageMagick-", "GraphicsMagick-"] as const;
|
||||
const WINDOWS_PROGRAM_FILES_TOOL_DIRS = ["ImageMagick", "GraphicsMagick"] as const;
|
||||
|
||||
const resolvedCacheStrict = new Map<string, string>();
|
||||
const resolvedCacheStandard = new Map<string, string>();
|
||||
@@ -44,6 +46,23 @@ function defaultIsExecutable(filePath: string): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
function collectWindowsProgramFilesToolDirs(programFilesRoot: string): string[] {
|
||||
const dirs = WINDOWS_PROGRAM_FILES_TOOL_DIRS.map((dir) => path.win32.join(programFilesRoot, dir));
|
||||
try {
|
||||
for (const entry of fs.readdirSync(programFilesRoot, { withFileTypes: true })) {
|
||||
if (
|
||||
entry.isDirectory() &&
|
||||
WINDOWS_PROGRAM_FILES_TOOL_DIR_PREFIXES.some((prefix) => entry.name.startsWith(prefix))
|
||||
) {
|
||||
dirs.push(path.win32.join(programFilesRoot, entry.name));
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Program Files can be unreadable in constrained contexts; static candidates still cover common installs.
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
let isExecutableFn: (filePath: string) => boolean = defaultIsExecutable;
|
||||
|
||||
/**
|
||||
@@ -55,6 +74,7 @@ function buildWindowsTrustedDirs(): readonly string[] {
|
||||
const { systemRoot } = getWindowsInstallRoots();
|
||||
dirs.push(path.win32.join(systemRoot, "System32"));
|
||||
dirs.push(path.win32.join(systemRoot, "SysWOW64"));
|
||||
dirs.push(path.win32.join(systemRoot, "System32", "WindowsPowerShell", "v1.0"));
|
||||
|
||||
for (const programFilesRoot of getWindowsProgramFilesRoots()) {
|
||||
// Trust the machine's validated Program Files roots rather than assuming C:.
|
||||
@@ -66,6 +86,16 @@ function buildWindowsTrustedDirs(): readonly string[] {
|
||||
return dirs;
|
||||
}
|
||||
|
||||
function buildWindowsStandardDirs(): readonly string[] {
|
||||
const { systemRoot } = getWindowsInstallRoots();
|
||||
const systemDriveRoot = path.win32.parse(systemRoot).root;
|
||||
const dirs = [path.win32.join(systemDriveRoot, "ProgramData", "chocolatey", "bin")];
|
||||
for (const programFilesRoot of getWindowsProgramFilesRoots()) {
|
||||
dirs.push(...collectWindowsProgramFilesToolDirs(programFilesRoot));
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the trusted-dir list for Unix (macOS, Linux, etc.), extending
|
||||
* UNIX_BASE_TRUSTED_DIRS with platform/environment-specific paths.
|
||||
@@ -106,9 +136,11 @@ let trustedDirsStandard: readonly string[] | null = null;
|
||||
|
||||
function getTrustedDirs(trust: SystemBinTrust): readonly string[] {
|
||||
if (process.platform === "win32") {
|
||||
// Windows does not currently widen "standard" beyond the registry-backed
|
||||
// system roots; both trust levels intentionally share the same set today.
|
||||
trustedDirsStrict ??= buildWindowsTrustedDirs();
|
||||
if (trust === "standard") {
|
||||
trustedDirsStandard ??= [...trustedDirsStrict, ...buildWindowsStandardDirs()];
|
||||
return trustedDirsStandard;
|
||||
}
|
||||
return trustedDirsStrict;
|
||||
}
|
||||
if (trust === "standard") {
|
||||
|
||||
@@ -281,7 +281,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
vi.doMock("../media/fetch.js", () => ({
|
||||
readRemoteMediaBuffer: readRemoteMediaBufferMock,
|
||||
}));
|
||||
vi.doMock("../media/ffmpeg-exec.js", () => ({
|
||||
vi.doMock("../media/media-services.js", () => ({
|
||||
runFfmpeg: runFfmpegMock,
|
||||
}));
|
||||
vi.doMock("../process/exec.js", () => ({
|
||||
|
||||
@@ -20,7 +20,7 @@ import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||
import { writeExternalFileWithinRoot } from "../infra/fs-safe.js";
|
||||
import { resolveProxyFetchFromEnv } from "../infra/net/proxy-fetch.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js";
|
||||
import { runFfmpeg } from "../media/ffmpeg-exec.js";
|
||||
import { runFfmpeg } from "../media/media-services.js";
|
||||
import { runExec } from "../process/exec.js";
|
||||
import { providerOperationRetryConfig } from "../provider-runtime/operation-retry.js";
|
||||
import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js";
|
||||
|
||||
@@ -10,7 +10,7 @@ vi.mock("./ffmpeg-exec.js", () => ({
|
||||
runFfmpeg: runFfmpegMock,
|
||||
}));
|
||||
|
||||
import { transcodeAudioBufferToOpus } from "./audio-transcode.js";
|
||||
import { transcodeAudioBuffer, transcodeAudioBufferToOpus } from "./audio-transcode.js";
|
||||
|
||||
type MockWithCalls = { mock: { calls: unknown[][] } };
|
||||
|
||||
@@ -156,3 +156,66 @@ describe("transcodeAudioBufferToOpus", () => {
|
||||
expect(capturedOutputPath ? existsSync(capturedOutputPath) : true).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("transcodeAudioBuffer", () => {
|
||||
afterEach(() => {
|
||||
runFfmpegMock.mockReset();
|
||||
});
|
||||
|
||||
it("returns noop-same-container when source and target containers match", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "mp3",
|
||||
targetExtension: ".mp3",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "noop-same-container" });
|
||||
});
|
||||
|
||||
it("returns no-recipe when no afconvert recipe is defined for the requested pair", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "mp3",
|
||||
targetExtension: "flac",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "no-recipe" });
|
||||
});
|
||||
|
||||
it("returns invalid-extension for an empty source extension", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "",
|
||||
targetExtension: "caf",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "invalid-extension" });
|
||||
});
|
||||
|
||||
it("returns invalid-extension for an empty target extension", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "mp3",
|
||||
targetExtension: "",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "invalid-extension" });
|
||||
});
|
||||
|
||||
it("rejects path-traversal style extensions", async () => {
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "../etc/passwd",
|
||||
targetExtension: "caf",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "invalid-extension" });
|
||||
});
|
||||
|
||||
it("returns platform-unsupported off-Darwin without invoking afconvert", async () => {
|
||||
if (process.platform === "darwin") {
|
||||
return;
|
||||
}
|
||||
const result = await transcodeAudioBuffer({
|
||||
audioBuffer: Buffer.from("payload"),
|
||||
sourceExtension: "mp3",
|
||||
targetExtension: "caf",
|
||||
});
|
||||
expect(result).toEqual({ ok: false, reason: "platform-unsupported" });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { writeExternalFileWithinRoot } from "../infra/fs-safe.js";
|
||||
import { withTempWorkspace } from "../infra/private-temp-workspace.js";
|
||||
import { tempWorkspaceSync, withTempWorkspace } from "../infra/private-temp-workspace.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js";
|
||||
import { runFfmpeg } from "./ffmpeg-exec.js";
|
||||
import { basenameFromAnyPath } from "./file-name.js";
|
||||
@@ -98,3 +99,94 @@ export async function transcodeAudioBufferToOpus(params: {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export type AudioContainerTranscodeOutcome =
|
||||
| { ok: true; buffer: Buffer }
|
||||
| {
|
||||
ok: false;
|
||||
reason:
|
||||
| "platform-unsupported"
|
||||
| "invalid-extension"
|
||||
| "noop-same-container"
|
||||
| "no-recipe"
|
||||
| "transcoder-failed";
|
||||
detail?: string;
|
||||
};
|
||||
|
||||
export async function transcodeAudioBuffer(params: {
|
||||
audioBuffer: Buffer;
|
||||
sourceExtension: string;
|
||||
targetExtension: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<AudioContainerTranscodeOutcome> {
|
||||
const source = normalizeContainerExt(params.sourceExtension);
|
||||
const target = normalizeContainerExt(params.targetExtension);
|
||||
if (!source || !target) {
|
||||
return { ok: false, reason: "invalid-extension" };
|
||||
}
|
||||
if (source === target) {
|
||||
return { ok: false, reason: "noop-same-container" };
|
||||
}
|
||||
const recipe = pickAfconvertRecipe(source, target);
|
||||
if (!recipe) {
|
||||
return { ok: false, reason: "no-recipe" };
|
||||
}
|
||||
if (process.platform !== "darwin") {
|
||||
return { ok: false, reason: "platform-unsupported" };
|
||||
}
|
||||
|
||||
const tmp = tempWorkspaceSync({
|
||||
rootDir: resolvePreferredOpenClawTmpDir(),
|
||||
prefix: "tts-transcode-",
|
||||
});
|
||||
const inPath = tmp.write(`in.${source}`, params.audioBuffer);
|
||||
const outPath = tmp.path(`out.${target}`);
|
||||
try {
|
||||
const result = await runAfconvert({
|
||||
args: [...recipe, inPath, outPath],
|
||||
timeoutMs: params.timeoutMs ?? 5000,
|
||||
});
|
||||
if (!result.ok) {
|
||||
return { ok: false, reason: "transcoder-failed", detail: result.detail };
|
||||
}
|
||||
return { ok: true, buffer: tmp.read(`out.${target}`) };
|
||||
} catch (err) {
|
||||
return { ok: false, reason: "transcoder-failed", detail: (err as Error).message };
|
||||
} finally {
|
||||
tmp.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeContainerExt(ext: string): string | undefined {
|
||||
const trimmed = ext.trim().toLowerCase().replace(/^\./, "");
|
||||
return /^[a-z0-9]{1,12}$/.test(trimmed) ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function pickAfconvertRecipe(_source: string, target: string): string[] | undefined {
|
||||
if (target === "caf") {
|
||||
// Opus-in-CAF matches native Messages voice memo attachments.
|
||||
return ["-f", "caff", "-d", "opus@24000", "-c", "1"];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function runAfconvert(params: {
|
||||
args: string[];
|
||||
timeoutMs: number;
|
||||
}): Promise<{ ok: true } | { ok: false; detail: string }> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("/usr/bin/afconvert", params.args, { stdio: "ignore" });
|
||||
const timer = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
resolve({ ok: false, detail: `timeout-${params.timeoutMs}ms` });
|
||||
}, params.timeoutMs);
|
||||
child.once("error", (err) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: false, detail: err.message });
|
||||
});
|
||||
child.once("exit", (code) => {
|
||||
clearTimeout(timer);
|
||||
resolve(code === 0 ? { ok: true } : { ok: false, detail: `exit-${code ?? "unknown"}` });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,16 +3,43 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveSystemBin } from "../infra/resolve-system-bin.js";
|
||||
import {
|
||||
convertHeicToJpeg,
|
||||
getImageMetadata,
|
||||
hasAlphaChannel,
|
||||
ImageProcessorUnavailableError,
|
||||
isImageProcessorUnavailableError,
|
||||
MAX_IMAGE_INPUT_PIXELS,
|
||||
resizeToJpeg,
|
||||
} from "./image-ops.js";
|
||||
import { createPngBufferWithDimensions } from "./test-helpers.js";
|
||||
|
||||
const PNG_1X1_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR4nGP8z8BQDwAFgwJ/lH3vWQAAAABJRU5ErkJggg==";
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=";
|
||||
|
||||
function isoBox(type: string, payload: Buffer): Buffer {
|
||||
const box = Buffer.alloc(8 + payload.length);
|
||||
box.writeUInt32BE(box.length, 0);
|
||||
box.write(type, 4, "ascii");
|
||||
payload.copy(box, 8);
|
||||
return box;
|
||||
}
|
||||
|
||||
function createHeifLikeBuffer(...sizes: Array<{ width: number; height: number }>): Buffer {
|
||||
const ftypPayload = Buffer.alloc(8);
|
||||
ftypPayload.write("heic", 0, "ascii");
|
||||
const ispeBoxes = sizes.map(({ width, height }) => {
|
||||
const ispePayload = Buffer.alloc(12);
|
||||
ispePayload.writeUInt32BE(width, 4);
|
||||
ispePayload.writeUInt32BE(height, 8);
|
||||
return isoBox("ispe", ispePayload);
|
||||
});
|
||||
const ipco = isoBox("ipco", Buffer.concat(ispeBoxes));
|
||||
const iprp = isoBox("iprp", ipco);
|
||||
const meta = isoBox("meta", Buffer.concat([Buffer.alloc(4), iprp]));
|
||||
return Buffer.concat([isoBox("ftyp", ftypPayload), meta]);
|
||||
}
|
||||
|
||||
describe("image input pixel guard", () => {
|
||||
const oversizedPng = createPngBufferWithDimensions({ width: 8_000, height: 4_000 });
|
||||
@@ -46,6 +73,30 @@ describe("image input pixel guard", () => {
|
||||
).rejects.toThrow(/pixel input limit/i);
|
||||
});
|
||||
|
||||
it("reads HEIF-style ISO BMFF dimensions without loading an image processor", async () => {
|
||||
await expect(
|
||||
getImageMetadata(createHeifLikeBuffer({ width: 640, height: 480 })),
|
||||
).resolves.toEqual({
|
||||
width: 640,
|
||||
height: 480,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects oversized HEIF-style ISO BMFF images before fallback tools run", async () => {
|
||||
const oversizedHeif = createHeifLikeBuffer(
|
||||
{ width: 64, height: 64 },
|
||||
{ width: 8_000, height: 4_000 },
|
||||
);
|
||||
await expect(getImageMetadata(oversizedHeif)).resolves.toBeNull();
|
||||
await expect(
|
||||
resizeToJpeg({
|
||||
buffer: oversizedHeif,
|
||||
maxSide: 2_048,
|
||||
quality: 80,
|
||||
}),
|
||||
).rejects.toThrow(/pixel input limit/i);
|
||||
});
|
||||
|
||||
it("fails closed when sips cannot determine image dimensions", async () => {
|
||||
const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND;
|
||||
process.env.OPENCLAW_IMAGE_BACKEND = "sips";
|
||||
@@ -66,6 +117,49 @@ describe("image input pixel guard", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("classifies image processor availability errors centrally", () => {
|
||||
expect(
|
||||
isImageProcessorUnavailableError(new ImageProcessorUnavailableError("resizeToJpeg")),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isImageProcessorUnavailableError(
|
||||
new Error("Optional dependency sharp is required for image attachment processing"),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("detects PNG alpha from headers without loading an image processor", async () => {
|
||||
const alphaPng = createPngBufferWithDimensions({ width: 1, height: 1 });
|
||||
const opaquePng = Buffer.from(alphaPng);
|
||||
opaquePng[25] = 2;
|
||||
|
||||
await expect(hasAlphaChannel(alphaPng)).resolves.toBe(true);
|
||||
await expect(hasAlphaChannel(opaquePng)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
const itIfFfmpeg = resolveSystemBin("ffmpeg", { trust: "standard" }) ? it : it.skip;
|
||||
|
||||
itIfFfmpeg("honors enlargement when the ffmpeg fallback is selected", async () => {
|
||||
const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND;
|
||||
process.env.OPENCLAW_IMAGE_BACKEND = "ffmpeg";
|
||||
try {
|
||||
const out = await resizeToJpeg({
|
||||
buffer: Buffer.from(PNG_1X1_BASE64, "base64"),
|
||||
maxSide: 4,
|
||||
quality: 90,
|
||||
withoutEnlargement: false,
|
||||
});
|
||||
|
||||
await expect(getImageMetadata(out)).resolves.toEqual({ width: 4, height: 4 });
|
||||
} finally {
|
||||
if (previousBackend === undefined) {
|
||||
delete process.env.OPENCLAW_IMAGE_BACKEND;
|
||||
} else {
|
||||
process.env.OPENCLAW_IMAGE_BACKEND = previousBackend;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const itIfMac = process.platform === "darwin" ? it : it.skip;
|
||||
|
||||
itIfMac("converts macOS-generated HEIC images to JPEG", async () => {
|
||||
@@ -81,7 +175,10 @@ describe("image input pixel guard", () => {
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
expect(result.status, result.stderr || result.stdout).toBe(0);
|
||||
if (result.status !== 0) {
|
||||
console.warn(`Skipping HEIC conversion fixture: ${result.stderr || result.stdout}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const jpeg = await convertHeicToJpeg(await fs.readFile(heicPath));
|
||||
|
||||
|
||||
+829
-103
File diff suppressed because it is too large
Load Diff
@@ -8,7 +8,7 @@ vi.mock("../infra/net/fetch-guard.js", () => ({
|
||||
fetchWithSsrFGuard: (...args: unknown[]) => fetchWithSsrFGuardMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock("./image-ops.js", () => ({
|
||||
vi.mock("./media-services.js", () => ({
|
||||
convertHeicToJpeg: (...args: unknown[]) => convertHeicToJpegMock(...args),
|
||||
}));
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
normalizeOptionalString,
|
||||
} from "../shared/string-coerce.js";
|
||||
import { canonicalizeBase64, estimateBase64DecodedBytes } from "./base64.js";
|
||||
import { convertHeicToJpeg } from "./image-ops.js";
|
||||
import { convertHeicToJpeg } from "./media-services.js";
|
||||
import { detectMime } from "./mime.js";
|
||||
import { extractPdfContent, type PdfExtractedImage } from "./pdf-extract.js";
|
||||
import { readResponseWithLimit } from "./read-response-with-limit.js";
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from "./audio-transcode.js";
|
||||
export * from "./ffmpeg-exec.js";
|
||||
export * from "./image-ops.js";
|
||||
export * from "./video-dimensions.js";
|
||||
@@ -241,15 +241,19 @@ describe("loadWebMedia", () => {
|
||||
|
||||
async function withUnavailableImageOptimizer<T>(fn: () => Promise<T>): Promise<T> {
|
||||
vi.resetModules();
|
||||
vi.doMock("./image-ops.js", () => ({
|
||||
vi.doMock("./media-services.js", () => ({
|
||||
convertHeicToJpeg: vi.fn(async (buffer: Buffer) => buffer),
|
||||
hasAlphaChannel: vi.fn(async () => {
|
||||
throw new Error(
|
||||
"Optional dependency sharp is required for image attachment processing | Cannot find package 'sharp' imported from image-ops.js",
|
||||
);
|
||||
}),
|
||||
isImageProcessorUnavailableError: (err: unknown) =>
|
||||
err instanceof Error && err.message.includes("Optional dependency sharp is required"),
|
||||
optimizeImageToPng: vi.fn(async () => {
|
||||
throw new Error("should not optimize png");
|
||||
throw new Error(
|
||||
"Optional dependency sharp is required for image attachment processing | Cannot find package 'sharp' imported from image-ops.js",
|
||||
);
|
||||
}),
|
||||
resizeToJpeg: vi.fn(async () => {
|
||||
throw new Error(
|
||||
@@ -260,7 +264,7 @@ describe("loadWebMedia", () => {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
vi.doUnmock("./image-ops.js");
|
||||
vi.doUnmock("./media-services.js");
|
||||
vi.resetModules();
|
||||
}
|
||||
}
|
||||
|
||||
+12
-24
@@ -9,12 +9,6 @@ import { resolveUserPath } from "../utils.js";
|
||||
import { maxBytesForKind, type MediaKind } from "./constants.js";
|
||||
import { readRemoteMediaBuffer } from "./fetch.js";
|
||||
import { basenameFromAnyPath, extnameFromAnyPath } from "./file-name.js";
|
||||
import {
|
||||
convertHeicToJpeg,
|
||||
hasAlphaChannel,
|
||||
optimizeImageToPng,
|
||||
resizeToJpeg,
|
||||
} from "./image-ops.js";
|
||||
import {
|
||||
assertLocalMediaAllowed,
|
||||
getDefaultLocalRoots,
|
||||
@@ -22,6 +16,13 @@ import {
|
||||
type LocalMediaAccessErrorCode,
|
||||
} from "./local-media-access.js";
|
||||
import { MediaReferenceError, resolveInboundMediaReference } from "./media-reference.js";
|
||||
import {
|
||||
convertHeicToJpeg,
|
||||
hasAlphaChannel,
|
||||
isImageProcessorUnavailableError,
|
||||
optimizeImageToPng,
|
||||
resizeToJpeg,
|
||||
} from "./media-services.js";
|
||||
import {
|
||||
detectMime,
|
||||
extensionForMime,
|
||||
@@ -229,23 +230,6 @@ function formatCapReduce(label: string, cap: number, size: number): string {
|
||||
return `${label} could not be reduced below ${formatMb(cap, 0)}MB (got ${formatMb(size)}MB)`;
|
||||
}
|
||||
|
||||
function isOptionalImageOptimizerUnavailable(err: unknown): boolean {
|
||||
const messages: string[] = [];
|
||||
let current: unknown = err;
|
||||
while (current instanceof Error) {
|
||||
messages.push(current.message);
|
||||
current = current.cause;
|
||||
}
|
||||
const detail = messages.join("\n").toLowerCase();
|
||||
return (
|
||||
detail.includes("optional dependency sharp is required") ||
|
||||
detail.includes("cannot find package 'sharp'") ||
|
||||
detail.includes('cannot find package "sharp"') ||
|
||||
detail.includes("cannot find module 'sharp'") ||
|
||||
detail.includes('cannot find module "sharp"')
|
||||
);
|
||||
}
|
||||
|
||||
function isHeicSource(opts: { contentType?: string; fileName?: string }): boolean {
|
||||
if (opts.contentType && HEIC_MIME_RE.test(opts.contentType.trim())) {
|
||||
return true;
|
||||
@@ -438,7 +422,7 @@ async function loadWebMediaInternal(
|
||||
optimized = await optimizeImageWithFallback({ buffer, cap, meta });
|
||||
} catch (err) {
|
||||
if (
|
||||
isOptionalImageOptimizerUnavailable(err) &&
|
||||
isImageProcessorUnavailableError(err) &&
|
||||
!isHeicSource(meta ?? {}) &&
|
||||
buffer.length <= cap
|
||||
) {
|
||||
@@ -724,6 +708,10 @@ export async function optimizeImageToJpeg(
|
||||
};
|
||||
}
|
||||
|
||||
if (isImageProcessorUnavailableError(firstResizeError)) {
|
||||
throw firstResizeError;
|
||||
}
|
||||
|
||||
const detail = errors.length > 0 ? `: ${errors.slice(0, 3).join("; ")}` : "";
|
||||
throw new Error(`Failed to optimize image${detail}`, { cause: firstResizeError });
|
||||
}
|
||||
|
||||
@@ -4,17 +4,40 @@
|
||||
*/
|
||||
|
||||
export * from "../media/audio.js";
|
||||
export * from "../media/audio-transcode.js";
|
||||
export * from "../media/base64.js";
|
||||
export * from "../media/constants.js";
|
||||
export * from "../media/fetch.js";
|
||||
export * from "../media/ffmpeg-exec.js";
|
||||
export * from "../media/ffmpeg-limits.js";
|
||||
export * from "../media/image-ops.js";
|
||||
export * from "../media/inbound-path-policy.js";
|
||||
export * from "../media/load-options.js";
|
||||
export * from "../media/local-media-access.js";
|
||||
export * from "../media/local-roots.js";
|
||||
export {
|
||||
IMAGE_REDUCE_QUALITY_STEPS,
|
||||
ImageProcessorUnavailableError,
|
||||
MAX_IMAGE_INPUT_PIXELS,
|
||||
buildImageResizeSideGrid,
|
||||
convertHeicToJpeg,
|
||||
getImageMetadata,
|
||||
hasAlphaChannel,
|
||||
isImageProcessorUnavailableError,
|
||||
normalizeExifOrientation,
|
||||
optimizeImageToPng,
|
||||
parseFfprobeCodecAndSampleRate,
|
||||
parseFfprobeCsvFields,
|
||||
parseFfprobeVideoDimensions,
|
||||
probeVideoDimensions,
|
||||
resizeToJpeg,
|
||||
resizeToPng,
|
||||
runFfmpeg,
|
||||
runFfprobe,
|
||||
transcodeAudioBuffer,
|
||||
transcodeAudioBufferToOpus,
|
||||
type AudioContainerTranscodeOutcome,
|
||||
type ImageMetadata,
|
||||
type MediaExecOptions,
|
||||
type VideoDimensions,
|
||||
} from "../media/media-services.js";
|
||||
export * from "../media/mime.js";
|
||||
export * from "../media/outbound-attachment.js";
|
||||
export * from "../media/png-encode.ts";
|
||||
@@ -24,7 +47,6 @@ export * from "../media/read-byte-stream-with-limit.js";
|
||||
export * from "../media/read-response-with-limit.js";
|
||||
export * from "../media/store.js";
|
||||
export * from "../media/temp-files.js";
|
||||
export * from "../media/video-dimensions.js";
|
||||
export { resolveChannelMediaMaxBytes } from "../channels/plugins/media-limits.js";
|
||||
export * from "./agent-media-payload.js";
|
||||
export * from "../media-understanding/audio-preflight.ts";
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { isVoiceCompatibleAudio } from "../../media/audio.js";
|
||||
import { mediaKindFromMime } from "../../media/constants.js";
|
||||
import { getImageMetadata, resizeToJpeg } from "../../media/image-ops.js";
|
||||
import { getImageMetadata, resizeToJpeg } from "../../media/media-services.js";
|
||||
import { detectMime } from "../../media/mime.js";
|
||||
import { loadWebMedia } from "../../media/web-media.js";
|
||||
import type { PluginRuntime } from "./types.js";
|
||||
|
||||
@@ -233,8 +233,8 @@ export type PluginRuntimeCore = {
|
||||
detectMime: typeof import("../../media/mime.js").detectMime;
|
||||
mediaKindFromMime: typeof import("../../media/constants.js").mediaKindFromMime;
|
||||
isVoiceCompatibleAudio: typeof import("../../media/audio.js").isVoiceCompatibleAudio;
|
||||
getImageMetadata: typeof import("../../media/image-ops.js").getImageMetadata;
|
||||
resizeToJpeg: typeof import("../../media/image-ops.js").resizeToJpeg;
|
||||
getImageMetadata: typeof import("../../media/media-services.js").getImageMetadata;
|
||||
resizeToJpeg: typeof import("../../media/media-services.js").resizeToJpeg;
|
||||
};
|
||||
tts: {
|
||||
textToSpeech: TextToSpeech;
|
||||
|
||||
Reference in New Issue
Block a user