mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
refactor(media): collapse attachment byte classification into media-core (#122168)
Attachment MIME gating and extraction dispatch used nine separate byte policies across five files; the allowlist could approve one MIME while dispatch observed another after a second sniff. @openclaw/media-core/attachment-classify now owns byte sniffing, text heuristics, extension mapping, and the seven attachment classes; the cache stores one classification that gates and dispatch share. Channel-declared MIME leads with the transport Content-Type as a byte-arbitrated hint, normalizeMimeType folds YAML/XML synonyms so existing configured allowlists keep matching, and input_file API callers keep declared-MIME precedence over cosmetic filenames. Net -135 production LOC. Proof: 2,153 focused tests across media-core/media-understanding/media plus boundary-contract and deadcode-mirror suites; ClawSweeper local review clean (round 5, zero findings); live Telegram default-turn probe green. Co-authored-by: Ayaan Zaidi <hi@obviy.us>
This commit is contained in:
Ayaan Zaidi
@@ -602,6 +602,7 @@ const config = {
|
||||
"packages/media-core": {
|
||||
entry: [
|
||||
"src/index.ts!",
|
||||
"src/attachment-classify.ts!",
|
||||
"src/base64.ts!",
|
||||
"src/constants.ts!",
|
||||
"src/content-length.ts!",
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"e93bb5a79e7a848c462ec88f8060fc87829dcd063da2c9698c27e19129754afc","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"}
|
||||
{"contentHash":"5a9d78a04eff220d33c4d2b7230c25f3d846ccddb4f2165967cf00fb948affa1","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"622cc8a24de6f41ff9d514399b034b7884d4f2e0f6f2bf75136640c80b1d8186","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"}
|
||||
{"contentHash":"8aef71ef2bd4a5984c9a0ddea4c4f1e6d86b9bf74504b50f8ec6a25ad65401cb","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"edc6c7ca91050af6dfceb136b3887f41548707f48115ed8c0ee34d097f6e524f","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"}
|
||||
{"contentHash":"df88650d813f6efbdffc5570647f183e69fa3015929c748b03e8b22bf0d916e6","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"4b3acf9c914456173c538f6635a95d797b96a1238eaab20314cd3b4c17809c23","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"}
|
||||
{"contentHash":"d11b2d65878b0886a3dc102c99a74fcd5e9bb80f27809ee810812f26ddde73cf","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"a9c548d441e74fe5c60af72f3225ff71ad75ba503554c7496ed174d9162d7f43","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"}
|
||||
{"contentHash":"3059df01f7a73ba9c511ce97a0ef358fd8109389f524c0a3914984745aa02db5","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"eb36acbe0e79d92d2acc8545bafb3efb998a73c6c198b0f89449a497a85448ca","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"}
|
||||
{"contentHash":"6fcaaa319bccd859ecbbe26278d3476ced29bdfed4e3ea0f05947c99d0e79bba","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"ecd4319e3dd1c8b390bc41546d784a99c1e640d3fed8d46ee041b2396f6b2538","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"}
|
||||
{"contentHash":"2d253351db6cb80191067f6ce1aefba74d35dbbf1816c2741078e91738d608d7","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"028ba363143c607ec08a4343df43d00cd6eee9a75aa5a56b1f3a8bef0f3e639f","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"}
|
||||
{"contentHash":"b9d274360639f0589b6036ba6d735d375a66edfe3f25fac3add96f5989c8bf8a","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"a9fedd2f64413f0f04751fc7551dbedb18e95573cf1bca3801de7defdf11a903","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"}
|
||||
{"contentHash":"5fb15ccec47ac420b0a1fb6d0769faeff1e14f3da1905c3c3a29c24887f1aaf1","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"db3caebf92c46468b996948e3a0ed6efeca1957a77704719fbf6bd81fc07095a","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"}
|
||||
{"contentHash":"8c220f857b5144582c560810b63948b14ba87524eff0aa7407d39ccb7df7de8f","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"10d52d83824b4b410912572ff36f135297fbeb8bb96328928fd30728e3f4546e","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"}
|
||||
{"contentHash":"f7dd2a2df426b3826fb9bf31a062004801675eb89c006508a91cd5f7713a739b","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"f1026c641b4a60fc6743fa07bced87548bf93ab1a7ac52076dde31e7ce6abe33","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"}
|
||||
{"contentHash":"6c93d74605b93456c5124fc15a3bb99d1a68734b9772a0514af3ded6fb647001","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"f6370f0532670140d2a80069c39f492f62d2cf79f6f0245635e03492c0570131","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"}
|
||||
{"contentHash":"f951f78a1ed81bc8c9eac5d2421e243abf28aac9297d21b2c174e22ce3fac045","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"5436af0315589c464d451a2cfdca71dc52b79a0906580c6656ebddb790792e48","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"}
|
||||
{"contentHash":"1291d3c53625156807274d0ddc752b3ae0b01bf937f0fdca504a879698d8194a","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"98be777b4b8309d6f8ff70db300b312482170eda016b93b94430ed137e932557","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"}
|
||||
{"contentHash":"c4ad78f6f23d67778f2d37cd7492e2c21832bd2622f56fa17d16f9b12fb1e955","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"54038626c9146bfc9bc4681fb345c39ab7de151b78c901fa125109c596cfd665","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"}
|
||||
{"contentHash":"2d76db26f7ce4188386d49fd13ed7daf5185cb22a6d2ac1fd9ade7288b6c7d0f","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"}
|
||||
|
||||
@@ -722,6 +722,9 @@
|
||||
"@openclaw/media-core": [
|
||||
"../dist/plugin-sdk/packages/media-core/src/index.d.ts"
|
||||
],
|
||||
"@openclaw/media-core/attachment-classify": [
|
||||
"../dist/plugin-sdk/packages/media-core/src/attachment-classify.d.ts"
|
||||
],
|
||||
"@openclaw/media-core/base64": [
|
||||
"../dist/plugin-sdk/packages/media-core/src/base64.d.ts"
|
||||
],
|
||||
|
||||
@@ -701,6 +701,9 @@
|
||||
"@openclaw/media-core": [
|
||||
"../../dist/plugin-sdk/packages/media-core/src/index.d.ts"
|
||||
],
|
||||
"@openclaw/media-core/attachment-classify": [
|
||||
"../../dist/plugin-sdk/packages/media-core/src/attachment-classify.d.ts"
|
||||
],
|
||||
"@openclaw/media-core/base64": [
|
||||
"../../dist/plugin-sdk/packages/media-core/src/base64.d.ts"
|
||||
],
|
||||
|
||||
@@ -19,6 +19,11 @@
|
||||
"import": "./dist/base64.mjs",
|
||||
"default": "./dist/base64.mjs"
|
||||
},
|
||||
"./attachment-classify": {
|
||||
"types": "./dist/attachment-classify.d.mts",
|
||||
"import": "./dist/attachment-classify.mjs",
|
||||
"default": "./dist/attachment-classify.mjs"
|
||||
},
|
||||
"./constants": {
|
||||
"types": "./dist/constants.d.mts",
|
||||
"import": "./dist/constants.mjs",
|
||||
@@ -65,6 +70,6 @@
|
||||
"file-type": "22.0.1"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsdown src/index.ts src/base64.ts src/constants.ts src/content-length.ts src/file-name.ts src/inbound-path-policy.ts src/inline-image-data-url.ts src/media-source-url.ts src/mime.ts src/read-byte-stream-with-limit.ts --no-config --platform node --format esm --dts --out-dir dist --clean"
|
||||
"build": "tsdown src/index.ts src/attachment-classify.ts src/base64.ts src/constants.ts src/content-length.ts src/file-name.ts src/inbound-path-policy.ts src/inline-image-data-url.ts src/media-source-url.ts src/mime.ts src/read-byte-stream-with-limit.ts --no-config --platform node --format esm --dts --out-dir dist --clean"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { attachmentClassFromMime, classifyAttachmentBytes } from "./attachment-classify.js";
|
||||
import { normalizeMimeType } from "./mime.js";
|
||||
|
||||
describe("attachmentClassFromMime", () => {
|
||||
it.each([
|
||||
["text/plain", "text"],
|
||||
["application/vnd.api+json", "text"],
|
||||
["application/pdf", "document"],
|
||||
["application/msword", "document"],
|
||||
["image/png", "image"],
|
||||
["audio/mpeg", "audio"],
|
||||
["video/mp4", "video"],
|
||||
["application/zip", "archive"],
|
||||
["application/octet-stream", "binary"],
|
||||
] as const)("classifies %s as %s", (mime, expected) => {
|
||||
expect(attachmentClassFromMime(mime)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyAttachmentBytes", () => {
|
||||
it("infers delimited text from otherwise untyped bytes", async () => {
|
||||
await expect(
|
||||
classifyAttachmentBytes({ buffer: Buffer.from("name,value\nopenclaw,1"), name: "data.bin" }),
|
||||
).resolves.toEqual({ mime: "text/csv", class: "text" });
|
||||
});
|
||||
|
||||
it("returns the UTF-16 charset with text classification", async () => {
|
||||
await expect(
|
||||
classifyAttachmentBytes({
|
||||
buffer: Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello", "utf16le")]),
|
||||
name: "notes.bin",
|
||||
}),
|
||||
).resolves.toEqual({ mime: "text/plain", class: "text", charset: "utf-16le" });
|
||||
});
|
||||
|
||||
it("keeps the charset when a BOM-less UTF-16 file resolves text by extension", async () => {
|
||||
await expect(
|
||||
classifyAttachmentBytes({
|
||||
buffer: Buffer.from("meeting notes for tomorrow", "utf16le"),
|
||||
name: "notes.txt",
|
||||
}),
|
||||
).resolves.toEqual({ mime: "text/plain", class: "text", charset: "utf-16le" });
|
||||
});
|
||||
|
||||
it("keeps byte-detected media ahead of a text filename", async () => {
|
||||
const png = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
);
|
||||
await expect(classifyAttachmentBytes({ buffer: png, name: "spoof.txt" })).resolves.toEqual({
|
||||
mime: "image/png",
|
||||
class: "image",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let a text filename override ZIP bytes", async () => {
|
||||
await expect(
|
||||
classifyAttachmentBytes({ buffer: Buffer.from("PK\u0003\u0004payload"), name: "spoof.txt" }),
|
||||
).resolves.toEqual({ mime: "application/zip", class: "archive" });
|
||||
});
|
||||
|
||||
it("keeps declared octet-stream content binary without a text extension", async () => {
|
||||
await expect(
|
||||
classifyAttachmentBytes({
|
||||
buffer: Buffer.from("printable but explicitly binary"),
|
||||
declaredMime: "application/octet-stream",
|
||||
name: "payload.bin",
|
||||
}),
|
||||
).resolves.toEqual({ mime: "application/octet-stream", class: "binary" });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["config.yaml", "application/yaml"],
|
||||
["payload.xml", "text/xml"],
|
||||
["debug.log", "text/plain"],
|
||||
["settings.ini", "text/plain"],
|
||||
] as const)("uses the canonical extension MIME for %s", async (name, mime) => {
|
||||
await expect(
|
||||
classifyAttachmentBytes({ buffer: Buffer.from("key=value"), name }),
|
||||
).resolves.toEqual({ mime, class: "text" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("mime synonym folding", () => {
|
||||
it("matches a configured text/yaml allowlist against classified .yaml files", async () => {
|
||||
const classified = await classifyAttachmentBytes({
|
||||
buffer: Buffer.from("key: value\nitems:\n - one\n", "utf8"),
|
||||
name: "config.yaml",
|
||||
});
|
||||
expect(classified.mime).toBe("application/yaml");
|
||||
expect(normalizeMimeType("text/yaml")).toBe(classified.mime);
|
||||
expect(normalizeMimeType("application/xml")).toBe("text/xml");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
import { detectMime, isZipContainerMime, mimeTypeFromFilePath, normalizeMimeType } from "./mime.js";
|
||||
|
||||
export type AttachmentClass =
|
||||
| "text"
|
||||
| "document"
|
||||
| "image"
|
||||
| "audio"
|
||||
| "video"
|
||||
| "archive"
|
||||
| "binary";
|
||||
type AttachmentCharset = "utf-16le" | "utf-16be";
|
||||
export type AttachmentClassification = {
|
||||
mime: string | undefined;
|
||||
class: AttachmentClass;
|
||||
charset?: AttachmentCharset;
|
||||
};
|
||||
|
||||
const TEXT_APPLICATION_MIME = /^application\/(?:json|javascript|xml|yaml|x-yaml)$/;
|
||||
const DOCUMENT_MIME =
|
||||
/^application\/(?:pdf|msword|x-cfb|vnd\.(?:apple\.(?:keynote|numbers|pages)|ms-.+|oasis\.opendocument\..+|openxmlformats-officedocument\..+))$/;
|
||||
const ARCHIVE_MIME =
|
||||
/^application\/(?:gzip|vnd\.rar|x-7z-compressed|x-gzip|x-rar-compressed|x-tar|x-zip-compressed|zip)$/;
|
||||
const WORDISH_CHAR = /[\p{L}\p{N}]/u;
|
||||
|
||||
export function attachmentClassFromMime(mime?: string | null): AttachmentClass {
|
||||
const normalized = normalizeMimeType(mime);
|
||||
if (!normalized) {
|
||||
return "binary";
|
||||
}
|
||||
if (
|
||||
normalized.startsWith("text/") ||
|
||||
TEXT_APPLICATION_MIME.test(normalized) ||
|
||||
normalized.endsWith("+json") ||
|
||||
normalized.endsWith("+xml")
|
||||
) {
|
||||
return "text";
|
||||
}
|
||||
if (normalized.startsWith("image/")) {
|
||||
return "image";
|
||||
}
|
||||
if (normalized.startsWith("audio/")) {
|
||||
return "audio";
|
||||
}
|
||||
if (normalized.startsWith("video/")) {
|
||||
return "video";
|
||||
}
|
||||
if (DOCUMENT_MIME.test(normalized)) {
|
||||
return "document";
|
||||
}
|
||||
return ARCHIVE_MIME.test(normalized) || isZipContainerMime(normalized) ? "archive" : "binary";
|
||||
}
|
||||
|
||||
function resolveUtf16Charset(buffer: Buffer): AttachmentCharset | undefined {
|
||||
if (buffer.length < 2) {
|
||||
return undefined;
|
||||
}
|
||||
const bom = buffer.readUInt16LE(0);
|
||||
if (bom === 0xfeff) {
|
||||
return "utf-16le";
|
||||
}
|
||||
if (bom === 0xfffe) {
|
||||
return "utf-16be";
|
||||
}
|
||||
const sampleLength = Math.min(buffer.length, 2048);
|
||||
let zeroEven = 0;
|
||||
let zeroOdd = 0;
|
||||
for (let index = 0; index < sampleLength; index += 1) {
|
||||
if (buffer[index] === 0) {
|
||||
if (index % 2 === 0) {
|
||||
zeroEven += 1;
|
||||
} else {
|
||||
zeroOdd += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((zeroEven + zeroOdd) / sampleLength <= 0.2) {
|
||||
return undefined;
|
||||
}
|
||||
return zeroOdd >= zeroEven ? "utf-16le" : "utf-16be";
|
||||
}
|
||||
|
||||
function textRatios(text: string): [printable: number, wordish: number] {
|
||||
let printable = 0;
|
||||
let control = 0;
|
||||
let wordish = 0;
|
||||
for (const char of text) {
|
||||
const code = char.codePointAt(0) ?? 0;
|
||||
if (code === 9 || code === 10 || code === 13 || code === 32) {
|
||||
printable += 1;
|
||||
wordish += 1;
|
||||
} else if (code < 32 || (code >= 0x7f && code <= 0x9f)) {
|
||||
control += 1;
|
||||
} else {
|
||||
printable += 1;
|
||||
wordish += Number(WORDISH_CHAR.test(char));
|
||||
}
|
||||
}
|
||||
const total = printable + control;
|
||||
return total === 0 ? [0, 0] : [printable / total, wordish / total];
|
||||
}
|
||||
|
||||
function looksLikeText(buffer: Buffer): boolean {
|
||||
if (buffer.length === 0) {
|
||||
return false;
|
||||
}
|
||||
const sample = buffer.subarray(0, Math.min(buffer.length, 4096));
|
||||
try {
|
||||
return textRatios(new TextDecoder("utf-8", { fatal: true }).decode(sample))[0] > 0.85;
|
||||
} catch {
|
||||
const [printable, wordish] = textRatios(new TextDecoder("windows-1252").decode(sample));
|
||||
return printable > 0.95 && wordish > 0.3;
|
||||
}
|
||||
}
|
||||
|
||||
export async function classifyAttachmentBytes(params: {
|
||||
buffer: Buffer;
|
||||
declaredMime?: string | null;
|
||||
/** Ordered fallback hints (e.g. transport Content-Type); bytes arbitrate. */
|
||||
additionalMimeHints?: readonly (string | null | undefined)[];
|
||||
name?: string | null;
|
||||
}): Promise<AttachmentClassification> {
|
||||
const mime = await detectMime({
|
||||
buffer: params.buffer,
|
||||
headerMime: params.declaredMime,
|
||||
additionalMimeHints: params.additionalMimeHints,
|
||||
filePath: params.name ?? undefined,
|
||||
});
|
||||
const detectedClass = attachmentClassFromMime(mime);
|
||||
const charset = resolveUtf16Charset(params.buffer);
|
||||
const hasUtf16Bom =
|
||||
params.buffer.length >= 2 &&
|
||||
(params.buffer.readUInt16LE(0) === 0xfeff || params.buffer.readUInt16LE(0) === 0xfffe);
|
||||
if (
|
||||
mime === "application/octet-stream" ||
|
||||
mime?.startsWith("application/vnd.") ||
|
||||
(detectedClass !== "binary" && !hasUtf16Bom)
|
||||
) {
|
||||
// Text resolved by extension can still be BOM-less UTF-16; dropping the
|
||||
// detected charset here would decode it downstream as UTF-8 mojibake.
|
||||
return detectedClass === "text" && charset
|
||||
? { mime, class: detectedClass, charset }
|
||||
: { mime, class: detectedClass };
|
||||
}
|
||||
const signature = params.buffer.length >= 4 ? params.buffer.readUInt32BE(0) : 0;
|
||||
if (signature === 0x504b0304 || signature === 0x504b0102 || signature === 0x504b0506) {
|
||||
return { mime, class: "archive" };
|
||||
}
|
||||
if (!charset && !looksLikeText(params.buffer)) {
|
||||
return { mime, class: "binary" };
|
||||
}
|
||||
const extensionMime = mimeTypeFromFilePath(params.name);
|
||||
const firstLine = new TextDecoder(charset ?? "utf-8")
|
||||
.decode(params.buffer.subarray(0, Math.min(params.buffer.length, 8192)))
|
||||
.split(/\r?\n/, 1)[0];
|
||||
const textMime =
|
||||
(attachmentClassFromMime(extensionMime) === "text" ? extensionMime : undefined) ??
|
||||
(firstLine?.includes(",")
|
||||
? "text/csv"
|
||||
: firstLine?.includes("\t")
|
||||
? "text/tab-separated-values"
|
||||
: "text/plain");
|
||||
return { mime: textMime, class: "text", ...(charset ? { charset } : {}) };
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
// Public barrel for media URL, MIME, path, and byte-stream helpers.
|
||||
|
||||
export * from "./base64.js";
|
||||
export * from "./attachment-classify.js";
|
||||
export * from "./constants.js";
|
||||
export * from "./content-length.js";
|
||||
export * from "./file-name.js";
|
||||
|
||||
@@ -90,9 +90,14 @@ const MIME_BY_EXT: Record<string, string> = {
|
||||
".aif": "audio/aiff",
|
||||
".aifc": "audio/aiff",
|
||||
".jpeg": "image/jpeg",
|
||||
".cfg": "text/plain",
|
||||
".conf": "text/plain",
|
||||
".env": "text/plain",
|
||||
".ini": "text/plain",
|
||||
".js": "text/javascript",
|
||||
".log": "text/plain",
|
||||
".htm": "text/html",
|
||||
".tsv": "text/tab-separated-values",
|
||||
".xml": "text/xml",
|
||||
".yml": "application/yaml",
|
||||
};
|
||||
@@ -145,20 +150,30 @@ const ZIP_CONTAINER_MIMES = new Set([
|
||||
"model/3mf",
|
||||
]);
|
||||
|
||||
function isZipContainerMime(mime: string): boolean {
|
||||
export function isZipContainerMime(mime: string): boolean {
|
||||
return mime.endsWith("+zip") || ZIP_CONTAINER_MIMES.has(mime);
|
||||
}
|
||||
|
||||
/** Normalizes MIME strings by dropping parameters, lowercasing, and folding APNG to PNG. */
|
||||
// Registered/legacy synonym pairs fold to one canonical spelling so configured
|
||||
// allowlists and byte classification always compare the same value; without
|
||||
// this an operator's existing text/yaml allowlist stops matching .yaml files.
|
||||
const MIME_SYNONYMS: Record<string, string> = {
|
||||
"image/apng": "image/png",
|
||||
"text/yaml": "application/yaml",
|
||||
"application/x-yaml": "application/yaml",
|
||||
"application/xml": "text/xml",
|
||||
};
|
||||
|
||||
/** Normalizes MIME strings by dropping parameters, lowercasing, and folding registered synonyms. */
|
||||
export function normalizeMimeType(mime?: string | null): string | undefined {
|
||||
if (!mime) {
|
||||
return undefined;
|
||||
}
|
||||
const cleaned = mime.split(";")[0]?.trim().toLowerCase();
|
||||
if (cleaned === "image/apng") {
|
||||
return "image/png";
|
||||
if (!cleaned) {
|
||||
return undefined;
|
||||
}
|
||||
return cleaned || undefined;
|
||||
return MIME_SYNONYMS[cleaned] ?? cleaned;
|
||||
}
|
||||
|
||||
/** Returns the bounded buffer prefix used for dependency MIME sniffing. */
|
||||
|
||||
@@ -158,6 +158,9 @@ export const EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS = {
|
||||
"../dist/plugin-sdk/packages/media-generation-core/src/*.d.ts",
|
||||
],
|
||||
"@openclaw/media-core": ["../dist/plugin-sdk/packages/media-core/src/index.d.ts"],
|
||||
"@openclaw/media-core/attachment-classify": [
|
||||
"../dist/plugin-sdk/packages/media-core/src/attachment-classify.d.ts",
|
||||
],
|
||||
"@openclaw/media-core/base64": ["../dist/plugin-sdk/packages/media-core/src/base64.d.ts"],
|
||||
"@openclaw/media-core/constants": ["../dist/plugin-sdk/packages/media-core/src/constants.d.ts"],
|
||||
"@openclaw/media-core/content-length": [
|
||||
|
||||
@@ -940,7 +940,7 @@ describe("sessions.files RPC handlers", () => {
|
||||
|
||||
it.each([
|
||||
{ format: "RTF", mimeType: "application/rtf", content: "{\\rtf1\\ansi hello}" },
|
||||
{ format: "XML", mimeType: "application/xml", content: '<?xml version="1.0"?><root/>' },
|
||||
{ format: "XML", mimeType: "text/xml", content: '<?xml version="1.0"?><root/>' },
|
||||
{ format: "WebVTT", mimeType: "text/vtt", content: "WEBVTT\n\n00:00.000 --> 00:01.000\nHi" },
|
||||
{ format: "vCard", mimeType: "text/vcard", content: "BEGIN:VCARD\nVERSION:4.0\nEND:VCARD\n" },
|
||||
{
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
// Applies media-understanding outputs to inbound message context, including
|
||||
// attachment normalization, provider execution, file text extraction, and echoing.
|
||||
import path from "node:path";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "@openclaw/normalization-core/string-coerce";
|
||||
attachmentClassFromMime,
|
||||
type AttachmentClassification,
|
||||
} from "@openclaw/media-core/attachment-classify";
|
||||
import { mimeTypeFromFilePath, normalizeMimeType } from "@openclaw/media-core/mime";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import pMap from "p-map";
|
||||
import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
|
||||
import {
|
||||
@@ -16,16 +17,9 @@ import type { MsgContext } from "../auto-reply/templating.js";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||
import { renderFileContextBlock } from "../media/file-context.js";
|
||||
import { extractFileContentFromSource, normalizeMimeType } from "../media/input-files.js";
|
||||
import { extractFileContentFromSource } from "../media/input-files.js";
|
||||
import { classifyMediaReferenceSource } from "../media/media-reference.js";
|
||||
import { runMediaCapability } from "./apply-capability.js";
|
||||
import {
|
||||
decodeTextSample,
|
||||
guessDelimitedMime,
|
||||
hasSuspiciousBinarySignal,
|
||||
looksLikeUtf8Text,
|
||||
resolveUtf16Charset,
|
||||
} from "./attachment-text-sniff.js";
|
||||
import { resolveAttachmentKind } from "./attachments.js";
|
||||
import { DEFAULT_ECHO_TRANSCRIPT_FORMAT, sendTranscriptEcho } from "./echo-transcript.js";
|
||||
import type { ExtractedFileImage } from "./extracted-file-images.js";
|
||||
@@ -73,31 +67,6 @@ const CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["image", "audio", "vid
|
||||
const AUDIO_ONLY_CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["audio"];
|
||||
const EMPTY_VOICE_NOTE_PLACEHOLDER =
|
||||
"[Voice note could not be transcribed because the audio attachment was too small]";
|
||||
const EXTRA_TEXT_MIMES = [
|
||||
"application/xml",
|
||||
"text/xml",
|
||||
"application/x-yaml",
|
||||
"text/yaml",
|
||||
"application/yaml",
|
||||
"application/javascript",
|
||||
"text/javascript",
|
||||
"text/tab-separated-values",
|
||||
];
|
||||
const TEXT_EXT_MIME = new Map<string, string>([
|
||||
[".csv", "text/csv"],
|
||||
[".tsv", "text/tab-separated-values"],
|
||||
[".txt", "text/plain"],
|
||||
[".md", "text/markdown"],
|
||||
[".log", "text/plain"],
|
||||
[".ini", "text/plain"],
|
||||
[".cfg", "text/plain"],
|
||||
[".conf", "text/plain"],
|
||||
[".env", "text/plain"],
|
||||
[".json", "application/json"],
|
||||
[".yaml", "text/yaml"],
|
||||
[".yml", "text/yaml"],
|
||||
[".xml", "application/xml"],
|
||||
]);
|
||||
|
||||
function appendFileBlocks(body: string | undefined, blocks: string[]): string {
|
||||
if (!blocks || blocks.length === 0) {
|
||||
@@ -111,14 +80,6 @@ function appendFileBlocks(body: string | undefined, blocks: string[]): string {
|
||||
return `${base}\n\n${suffix}`.trim();
|
||||
}
|
||||
|
||||
function resolveTextMimeFromName(name?: string): string | undefined {
|
||||
if (!name) {
|
||||
return undefined;
|
||||
}
|
||||
const ext = normalizeLowercaseStringOrEmpty(path.extname(name));
|
||||
return TEXT_EXT_MIME.get(ext);
|
||||
}
|
||||
|
||||
function buildSyntheticSkippedAudioOutputs(
|
||||
decisions: MediaUnderstandingDecision[],
|
||||
): MediaUnderstandingOutput[] {
|
||||
@@ -145,42 +106,6 @@ function buildSyntheticSkippedAudioOutputs(
|
||||
});
|
||||
}
|
||||
|
||||
function isBinaryMediaMime(mime?: string): boolean {
|
||||
if (!mime) {
|
||||
return false;
|
||||
}
|
||||
if (mime.startsWith("image/") || mime.startsWith("audio/") || mime.startsWith("video/")) {
|
||||
return true;
|
||||
}
|
||||
if (mime === "application/octet-stream") {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
mime === "application/zip" ||
|
||||
mime === "application/x-zip-compressed" ||
|
||||
mime === "application/gzip" ||
|
||||
mime === "application/x-gzip" ||
|
||||
mime === "application/x-rar-compressed" ||
|
||||
mime === "application/x-7z-compressed" ||
|
||||
mime === "application/msword" ||
|
||||
mime === "application/x-cfb"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (mime.endsWith("+zip")) {
|
||||
return true;
|
||||
}
|
||||
if (mime.startsWith("application/vnd.")) {
|
||||
// Keep vendor +json/+xml payloads eligible for text extraction while
|
||||
// treating the common binary vendor family (Office, archives, etc.) as binary.
|
||||
if (mime.endsWith("+json") || mime.endsWith("+xml")) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
type ClassifiedFileAttachment = {
|
||||
outcome: FileAttachmentOutcome;
|
||||
filename?: string;
|
||||
@@ -213,7 +138,9 @@ async function classifyFileAttachment(params: {
|
||||
if (skipAttachmentIndexes?.has(attachment.index)) {
|
||||
return { outcome: { kind: "claimed-elsewhere" } };
|
||||
}
|
||||
const forcedTextMime = resolveTextMimeFromName(attachmentFilename ?? "");
|
||||
const extensionMime = mimeTypeFromFilePath(attachmentFilename);
|
||||
const forcedTextMime =
|
||||
attachmentClassFromMime(extensionMime) === "text" ? extensionMime : undefined;
|
||||
const kind = forcedTextMime ? "document" : resolveAttachmentKind(attachment);
|
||||
if (!forcedTextMime && (kind === "image" || kind === "video" || kind === "audio")) {
|
||||
return { outcome: { kind: "claimed-elsewhere" } };
|
||||
@@ -243,45 +170,31 @@ async function classifyFileAttachment(params: {
|
||||
return { outcome: { kind: "read-failure" }, filename: attachmentFilename };
|
||||
}
|
||||
const filename = bufferResult?.fileName;
|
||||
const nameHint = filename ?? attachmentFilename;
|
||||
const forcedTextMimeResolved = forcedTextMime ?? resolveTextMimeFromName(nameHint ?? "");
|
||||
const rawMime = bufferResult?.mime ?? attachment.mime;
|
||||
const normalizedRawMime = normalizeMimeType(rawMime);
|
||||
const classification: AttachmentClassification = bufferResult.classification;
|
||||
// Marker mime prefers the sender-declared type; never the name-forced text mime,
|
||||
// which would mislabel binary bytes inside a text-named file as a text format.
|
||||
// Both candidates pass strict token validation so raw header text never
|
||||
// reaches model context; undefined drops the mime from block and marker.
|
||||
const binaryMime =
|
||||
sanitizeMimeType(normalizeMimeType(attachment.mime)) ?? sanitizeMimeType(normalizedRawMime);
|
||||
if (!forcedTextMimeResolved && isBinaryMediaMime(normalizedRawMime)) {
|
||||
sanitizeMimeType(normalizeMimeType(attachment.mime)) ?? sanitizeMimeType(classification.mime);
|
||||
if (
|
||||
classification.class !== "text" &&
|
||||
!(classification.class === "document" && classification.mime === "application/pdf")
|
||||
) {
|
||||
return {
|
||||
outcome: { kind: "unsupported-format", mime: binaryMime },
|
||||
filename,
|
||||
mimeType: binaryMime,
|
||||
};
|
||||
}
|
||||
if (hasSuspiciousBinarySignal(bufferResult?.buffer)) {
|
||||
return {
|
||||
outcome: { kind: "unsupported-format", mime: binaryMime },
|
||||
filename,
|
||||
mimeType: binaryMime,
|
||||
};
|
||||
}
|
||||
const utf16Charset = resolveUtf16Charset(bufferResult?.buffer);
|
||||
const textSample = decodeTextSample(bufferResult?.buffer);
|
||||
// Do not coerce real PDFs into text/plain via printable-byte heuristics.
|
||||
// PDFs have a dedicated extraction path in extractFileContentFromSource.
|
||||
const allowTextHeuristic = normalizedRawMime !== "application/pdf";
|
||||
const textLike =
|
||||
allowTextHeuristic && (Boolean(utf16Charset) || looksLikeUtf8Text(bufferResult?.buffer));
|
||||
const guessedDelimited = textLike ? guessDelimitedMime(textSample) : undefined;
|
||||
const textHint =
|
||||
forcedTextMimeResolved ?? guessedDelimited ?? (textLike ? "text/plain" : undefined);
|
||||
const mimeType = sanitizeMimeType(textHint ?? normalizeMimeType(rawMime));
|
||||
// Log when MIME type is overridden from non-text to text for auditability
|
||||
if (textHint && rawMime && !rawMime.startsWith("text/")) {
|
||||
const mimeType = sanitizeMimeType(classification.mime);
|
||||
if (
|
||||
classification.class === "text" &&
|
||||
attachment.mime &&
|
||||
normalizeMimeType(attachment.mime) !== classification.mime
|
||||
) {
|
||||
logVerbose(
|
||||
`media: MIME override from "${rawMime}" to "${textHint}" for index=${attachment.index}`,
|
||||
`media: MIME override from "${attachment.mime}" to "${classification.mime}" for index=${attachment.index}`,
|
||||
);
|
||||
}
|
||||
if (!mimeType) {
|
||||
@@ -291,13 +204,8 @@ async function classifyFileAttachment(params: {
|
||||
return { outcome: { kind: "unsupported-format" }, filename };
|
||||
}
|
||||
const allowedMimes = new Set(limits.allowedMimes);
|
||||
if (!limits.allowedMimesConfigured) {
|
||||
for (const extra of EXTRA_TEXT_MIMES) {
|
||||
allowedMimes.add(extra);
|
||||
}
|
||||
if (mimeType.startsWith("text/")) {
|
||||
allowedMimes.add(mimeType);
|
||||
}
|
||||
if (!limits.allowedMimesConfigured && classification.class === "text") {
|
||||
allowedMimes.add(mimeType);
|
||||
}
|
||||
if (!allowedMimes.has(mimeType)) {
|
||||
if (shouldLogVerbose()) {
|
||||
@@ -315,17 +223,17 @@ async function classifyFileAttachment(params: {
|
||||
}
|
||||
let extracted: Awaited<ReturnType<typeof extractFileContentFromSource>>;
|
||||
try {
|
||||
const mediaType = utf16Charset ? `${mimeType}; charset=${utf16Charset}` : mimeType;
|
||||
const { allowedMimesConfigured: _allowedMimesConfigured, ...baseLimits } = limits;
|
||||
extracted = await extractFileContentFromSource({
|
||||
source: {
|
||||
type: "base64",
|
||||
data: bufferResult.buffer.toString("base64"),
|
||||
mediaType,
|
||||
mediaType: mimeType,
|
||||
filename: bufferResult.fileName,
|
||||
},
|
||||
limits: { ...baseLimits, allowedMimes },
|
||||
config: cfg,
|
||||
classification,
|
||||
});
|
||||
} catch (err) {
|
||||
if (shouldLogVerbose()) {
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
// Byte-level text detection for inbound attachments: charset resolution,
|
||||
// legacy-codepage heuristics, archive signatures, and delimiter guessing.
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
|
||||
export function resolveUtf16Charset(buffer?: Buffer): "utf-16le" | "utf-16be" | undefined {
|
||||
// Some chat attachments arrive as UTF-16 without a reliable MIME charset; the
|
||||
// BOM and zero-byte distribution are enough to select a safe decoder.
|
||||
if (!buffer || buffer.length < 2) {
|
||||
return undefined;
|
||||
}
|
||||
const b0 = buffer[0];
|
||||
const b1 = buffer[1];
|
||||
if (b0 === 0xff && b1 === 0xfe) {
|
||||
return "utf-16le";
|
||||
}
|
||||
if (b0 === 0xfe && b1 === 0xff) {
|
||||
return "utf-16be";
|
||||
}
|
||||
const sampleLen = Math.min(buffer.length, 2048);
|
||||
let zeroEven = 0;
|
||||
let zeroOdd = 0;
|
||||
for (let i = 0; i < sampleLen; i += 1) {
|
||||
if (buffer[i] !== 0) {
|
||||
continue;
|
||||
}
|
||||
if (i % 2 === 0) {
|
||||
zeroEven += 1;
|
||||
} else {
|
||||
zeroOdd += 1;
|
||||
}
|
||||
}
|
||||
const zeroCount = zeroEven + zeroOdd;
|
||||
if (zeroCount / sampleLen > 0.2) {
|
||||
return zeroOdd >= zeroEven ? "utf-16le" : "utf-16be";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const WORDISH_CHAR = /[\p{L}\p{N}]/u;
|
||||
const CP1252_MAP: Array<string | undefined> = [
|
||||
"\u20ac",
|
||||
undefined,
|
||||
"\u201a",
|
||||
"\u0192",
|
||||
"\u201e",
|
||||
"\u2026",
|
||||
"\u2020",
|
||||
"\u2021",
|
||||
"\u02c6",
|
||||
"\u2030",
|
||||
"\u0160",
|
||||
"\u2039",
|
||||
"\u0152",
|
||||
undefined,
|
||||
"\u017d",
|
||||
undefined,
|
||||
undefined,
|
||||
"\u2018",
|
||||
"\u2019",
|
||||
"\u201c",
|
||||
"\u201d",
|
||||
"\u2022",
|
||||
"\u2013",
|
||||
"\u2014",
|
||||
"\u02dc",
|
||||
"\u2122",
|
||||
"\u0161",
|
||||
"\u203a",
|
||||
"\u0153",
|
||||
undefined,
|
||||
"\u017e",
|
||||
"\u0178",
|
||||
];
|
||||
|
||||
function decodeLegacyText(buffer: Buffer): string {
|
||||
let output = "";
|
||||
for (const byte of buffer) {
|
||||
if (byte >= 0x80 && byte <= 0x9f) {
|
||||
const mapped = CP1252_MAP[byte - 0x80];
|
||||
output += mapped ?? String.fromCharCode(byte);
|
||||
continue;
|
||||
}
|
||||
output += String.fromCharCode(byte);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function getTextStats(text: string): { printableRatio: number; wordishRatio: number } {
|
||||
if (!text) {
|
||||
return { printableRatio: 0, wordishRatio: 0 };
|
||||
}
|
||||
let printable = 0;
|
||||
let control = 0;
|
||||
let wordish = 0;
|
||||
for (const char of text) {
|
||||
const code = char.codePointAt(0) ?? 0;
|
||||
if (code === 9 || code === 10 || code === 13 || code === 32) {
|
||||
printable += 1;
|
||||
wordish += 1;
|
||||
continue;
|
||||
}
|
||||
if (code < 32 || (code >= 0x7f && code <= 0x9f)) {
|
||||
control += 1;
|
||||
continue;
|
||||
}
|
||||
printable += 1;
|
||||
if (WORDISH_CHAR.test(char)) {
|
||||
wordish += 1;
|
||||
}
|
||||
}
|
||||
const total = printable + control;
|
||||
if (total === 0) {
|
||||
return { printableRatio: 0, wordishRatio: 0 };
|
||||
}
|
||||
return { printableRatio: printable / total, wordishRatio: wordish / total };
|
||||
}
|
||||
|
||||
function isMostlyPrintable(text: string): boolean {
|
||||
return getTextStats(text).printableRatio > 0.85;
|
||||
}
|
||||
|
||||
function looksLikeLegacyTextBytes(buffer: Buffer): boolean {
|
||||
if (buffer.length === 0) {
|
||||
return false;
|
||||
}
|
||||
const text = decodeLegacyText(buffer);
|
||||
const { printableRatio, wordishRatio } = getTextStats(text);
|
||||
return printableRatio > 0.95 && wordishRatio > 0.3;
|
||||
}
|
||||
|
||||
export function looksLikeUtf8Text(buffer?: Buffer): boolean {
|
||||
if (!buffer || buffer.length === 0) {
|
||||
return false;
|
||||
}
|
||||
const sample = buffer.subarray(0, Math.min(buffer.length, 4096));
|
||||
try {
|
||||
const text = new TextDecoder("utf-8", { fatal: true }).decode(sample);
|
||||
return isMostlyPrintable(text);
|
||||
} catch {
|
||||
return looksLikeLegacyTextBytes(sample);
|
||||
}
|
||||
}
|
||||
|
||||
export function hasSuspiciousBinarySignal(buffer?: Buffer): boolean {
|
||||
if (!buffer || buffer.length === 0) {
|
||||
return false;
|
||||
}
|
||||
const sample = buffer.subarray(0, Math.min(buffer.length, 4096));
|
||||
if (sample.length < 4 || sample[0] !== 0x50 || sample[1] !== 0x4b) {
|
||||
return false;
|
||||
}
|
||||
const signature =
|
||||
(expectDefined(sample[2], "sample entry at 2") << 8) |
|
||||
expectDefined(sample[3], "sample entry at 3");
|
||||
// Cover the ZIP local-header, central-directory, and empty-archive markers
|
||||
// so archive payloads cannot slip past text coercion when MIME detection is weak.
|
||||
return signature === 0x0304 || signature === 0x0102 || signature === 0x0506;
|
||||
}
|
||||
|
||||
export function decodeTextSample(buffer?: Buffer): string {
|
||||
if (!buffer || buffer.length === 0) {
|
||||
return "";
|
||||
}
|
||||
const sample = buffer.subarray(0, Math.min(buffer.length, 8192));
|
||||
const utf16Charset = resolveUtf16Charset(sample);
|
||||
if (utf16Charset === "utf-16be") {
|
||||
const swapped = Buffer.alloc(sample.length);
|
||||
for (let i = 0; i + 1 < sample.length; i += 2) {
|
||||
swapped[i] = expectDefined(sample[i + 1], "UTF-16BE low byte");
|
||||
swapped[i + 1] = expectDefined(sample[i], "UTF-16BE high byte");
|
||||
}
|
||||
return new TextDecoder("utf-16le").decode(swapped);
|
||||
}
|
||||
if (utf16Charset === "utf-16le") {
|
||||
return new TextDecoder("utf-16le").decode(sample);
|
||||
}
|
||||
return new TextDecoder("utf-8").decode(sample);
|
||||
}
|
||||
|
||||
export function guessDelimitedMime(text: string): string | undefined {
|
||||
if (!text) {
|
||||
return undefined;
|
||||
}
|
||||
const line = text.split(/\r?\n/)[0] ?? "";
|
||||
const tabs = (line.match(/\t/g) ?? []).length;
|
||||
const commas = (line.match(/,/g) ?? []).length;
|
||||
if (commas > 0) {
|
||||
return "text/csv";
|
||||
}
|
||||
if (tabs > 0) {
|
||||
return "text/tab-separated-values";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -3,11 +3,14 @@
|
||||
import { realpathSync, statSync } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
classifyAttachmentBytes,
|
||||
type AttachmentClassification,
|
||||
} from "@openclaw/media-core/attachment-classify";
|
||||
import {
|
||||
isInboundPathAllowed,
|
||||
mergeInboundPathRoots,
|
||||
} from "@openclaw/media-core/inbound-path-policy";
|
||||
import { detectMime } from "@openclaw/media-core/mime";
|
||||
import { MediaUnderstandingSkipError } from "../../packages/media-understanding-common/src/errors.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||
@@ -31,6 +34,7 @@ import type { MediaAttachment } from "./types.js";
|
||||
|
||||
type MediaBufferResult = {
|
||||
buffer: Buffer;
|
||||
classification: AttachmentClassification;
|
||||
mime?: string;
|
||||
fileName: string;
|
||||
size: number;
|
||||
@@ -57,9 +61,7 @@ type AttachmentCacheEntry = {
|
||||
attachment: MediaAttachment;
|
||||
resolvedPath?: string;
|
||||
statSize?: number;
|
||||
buffer?: Buffer;
|
||||
bufferMime?: string;
|
||||
bufferFileName?: string;
|
||||
bufferResult?: MediaBufferResult;
|
||||
tempPath?: string;
|
||||
tempCleanup?: () => Promise<void>;
|
||||
localResolutionAttempted?: boolean;
|
||||
@@ -83,7 +85,9 @@ function remoteFetchUrl(url: string | undefined): string | undefined {
|
||||
|
||||
function concreteMime(mime: string | undefined): string | undefined {
|
||||
const normalized = mime?.trim();
|
||||
if (!normalized || normalized.endsWith("/*")) {
|
||||
// octet-stream is a non-answer, not a concrete type; a generic download
|
||||
// header must not shadow the attachment fact's declared MIME.
|
||||
if (!normalized || normalized.endsWith("/*") || normalized === "application/octet-stream") {
|
||||
return undefined;
|
||||
}
|
||||
return normalized;
|
||||
@@ -164,19 +168,14 @@ export class MediaAttachmentCache {
|
||||
}): Promise<MediaBufferResult> {
|
||||
const entry = await this.ensureEntry(params.attachmentIndex);
|
||||
const url = remoteFetchUrl(entry.attachment.url);
|
||||
if (entry.buffer) {
|
||||
if (entry.buffer.length > params.maxBytes) {
|
||||
if (entry.bufferResult) {
|
||||
if (entry.bufferResult.size > params.maxBytes) {
|
||||
throw new MediaUnderstandingSkipError(
|
||||
"maxBytes",
|
||||
`Attachment ${params.attachmentIndex + 1} exceeds maxBytes ${params.maxBytes}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
buffer: entry.buffer,
|
||||
mime: entry.bufferMime,
|
||||
fileName: entry.bufferFileName ?? `media-${params.attachmentIndex + 1}`,
|
||||
size: entry.buffer.length,
|
||||
};
|
||||
return entry.bufferResult;
|
||||
}
|
||||
|
||||
if (entry.resolvedPath) {
|
||||
@@ -223,20 +222,22 @@ export class MediaAttachmentCache {
|
||||
ssrfPolicy: this.ssrfPolicy,
|
||||
retry: REMOTE_MEDIA_FETCH_RETRY,
|
||||
});
|
||||
entry.buffer = fetched.buffer;
|
||||
entry.bufferMime = await detectMime({
|
||||
const classification = await classifyAttachmentBytes({
|
||||
buffer: fetched.buffer,
|
||||
filePath: fetched.fileName ?? url,
|
||||
headerMime: concreteMime(entry.attachment.mime),
|
||||
name: fetched.fileName ?? url,
|
||||
// Channel-declared MIME leads; the transport Content-Type stays a
|
||||
// secondary hint so byte detection can refine stale declarations.
|
||||
declaredMime: concreteMime(entry.attachment.mime),
|
||||
additionalMimeHints: [fetched.contentType],
|
||||
});
|
||||
entry.bufferFileName = fetched.fileName ?? `media-${params.attachmentIndex + 1}`;
|
||||
return {
|
||||
entry.bufferResult = {
|
||||
buffer: fetched.buffer,
|
||||
mime: entry.bufferMime,
|
||||
fileName: entry.bufferFileName,
|
||||
classification,
|
||||
mime: classification.mime,
|
||||
fileName: fetched.fileName ?? `media-${params.attachmentIndex + 1}`,
|
||||
size: fetched.buffer.length,
|
||||
};
|
||||
return entry.bufferResult;
|
||||
} catch (err) {
|
||||
if (err instanceof MediaFetchError && err.code === "max_bytes") {
|
||||
throw new MediaUnderstandingSkipError(
|
||||
@@ -275,21 +276,19 @@ export class MediaAttachmentCache {
|
||||
maxBytes: params.maxBytes,
|
||||
});
|
||||
entry.resolvedPath = filePath;
|
||||
entry.buffer = buffer;
|
||||
entry.bufferMime =
|
||||
entry.bufferMime ??
|
||||
(await detectMime({
|
||||
buffer,
|
||||
filePath,
|
||||
headerMime: concreteMime(entry.attachment.mime),
|
||||
}));
|
||||
entry.bufferFileName = path.basename(filePath) || `media-${params.attachmentIndex + 1}`;
|
||||
return {
|
||||
const classification = await classifyAttachmentBytes({
|
||||
buffer,
|
||||
mime: entry.bufferMime,
|
||||
fileName: entry.bufferFileName,
|
||||
name: filePath,
|
||||
declaredMime: concreteMime(entry.attachment.mime),
|
||||
});
|
||||
entry.bufferResult = {
|
||||
buffer,
|
||||
classification,
|
||||
mime: classification.mime,
|
||||
fileName: path.basename(filePath) || `media-${params.attachmentIndex + 1}`,
|
||||
size: buffer.length,
|
||||
};
|
||||
return entry.bufferResult;
|
||||
}
|
||||
|
||||
private recordRecoverableLocalError(entry: AttachmentCacheEntry, err: unknown): boolean {
|
||||
@@ -367,7 +366,7 @@ export class MediaAttachmentCache {
|
||||
}
|
||||
|
||||
if (entry.tempPath) {
|
||||
if (params.maxBytes && entry.buffer && entry.buffer.length > params.maxBytes) {
|
||||
if (params.maxBytes && entry.bufferResult && entry.bufferResult.size > params.maxBytes) {
|
||||
throw new MediaUnderstandingSkipError(
|
||||
"maxBytes",
|
||||
`Attachment ${params.attachmentIndex + 1} exceeds maxBytes ${params.maxBytes}`,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Image input normalization converts HEIC/HEIF payloads through the shared
|
||||
// input-file media path before provider execution.
|
||||
import { extractImageContentFromSource, normalizeMimeType } from "../media/input-files.js";
|
||||
import { normalizeMimeType } from "@openclaw/media-core/mime";
|
||||
import { extractImageContentFromSource } from "../media/input-files.js";
|
||||
import { DEFAULT_MAX_BYTES } from "./defaults.constants.js";
|
||||
|
||||
const HEIC_MIME_RE = /^image\/hei[cf]$/i;
|
||||
|
||||
@@ -110,6 +110,7 @@ describe("media understanding attachments SSRF", () => {
|
||||
});
|
||||
expect(result).toStrictEqual({
|
||||
buffer: Buffer.from("image"),
|
||||
classification: { mime: "image/jpeg", class: "image" },
|
||||
mime: "image/jpeg",
|
||||
fileName: "file.jpg",
|
||||
size: 5,
|
||||
@@ -139,6 +140,53 @@ describe("media understanding attachments SSRF", () => {
|
||||
expect(result.fileName).toBe("image.png");
|
||||
});
|
||||
|
||||
it("keeps the attachment's concrete MIME when the download reports octet-stream", async () => {
|
||||
const url = "http://198.18.0.153/export";
|
||||
const fetchSpy = vi.fn().mockResolvedValue(
|
||||
new Response('{"report":"q3"}', {
|
||||
headers: { "content-type": "application/octet-stream" },
|
||||
}),
|
||||
);
|
||||
globalThis.fetch = withFetchPreconnect(fetchSpy);
|
||||
const cache = new MediaAttachmentCache([{ index: 0, url, mime: "application/json" }], {
|
||||
ssrfPolicy: { allowRfc2544BenchmarkRange: true },
|
||||
});
|
||||
|
||||
const result = await cache.getBuffer({
|
||||
attachmentIndex: 0,
|
||||
maxBytes: 1024,
|
||||
timeoutMs: 1000,
|
||||
});
|
||||
|
||||
// A generic download header is a non-answer; the declared concrete type
|
||||
// must survive so allowlists and extraction see application/json.
|
||||
expect(result.classification.mime).toBe("application/json");
|
||||
expect(result.classification.class).toBe("text");
|
||||
});
|
||||
|
||||
it("prefers the channel-declared MIME over a generic fetched text header", async () => {
|
||||
const url = "http://198.18.0.153/export.data";
|
||||
const fetchSpy = vi.fn().mockResolvedValue(
|
||||
new Response('{"report":"q4"}', {
|
||||
headers: { "content-type": "text/plain" },
|
||||
}),
|
||||
);
|
||||
globalThis.fetch = withFetchPreconnect(fetchSpy);
|
||||
const cache = new MediaAttachmentCache([{ index: 0, url, mime: "application/json" }], {
|
||||
ssrfPolicy: { allowRfc2544BenchmarkRange: true },
|
||||
});
|
||||
|
||||
const result = await cache.getBuffer({
|
||||
attachmentIndex: 0,
|
||||
maxBytes: 1024,
|
||||
timeoutMs: 1000,
|
||||
});
|
||||
|
||||
// Channel declaration outranks transport Content-Type: a JSON-only
|
||||
// allowlist must keep matching when a proxy relabels the download.
|
||||
expect(result.classification.mime).toBe("application/json");
|
||||
});
|
||||
|
||||
it("reads local attachments inside configured roots", async () => {
|
||||
await withLocalAttachmentCache("openclaw-media-cache-allowed-", async ({ cache }) => {
|
||||
const result = await cache.getBuffer({ attachmentIndex: 0, maxBytes: 1024, timeoutMs: 1000 });
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
// Regression: input_file callers declare their MIME; a cosmetic filename must
|
||||
// not reroute classification past an operator-configured allowlist.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { extractFileContentFromSource, resolveInputFileLimits } from "./input-files.js";
|
||||
|
||||
describe("extractFileContentFromSource", () => {
|
||||
it("keeps the declared MIME when the filename suggests plain text", async () => {
|
||||
const payload = JSON.stringify({ report: "q3", revenue: 12345 });
|
||||
const limits = resolveInputFileLimits({ allowedMimes: ["application/json"] });
|
||||
|
||||
const result = await extractFileContentFromSource({
|
||||
source: {
|
||||
type: "base64",
|
||||
data: Buffer.from(payload, "utf8").toString("base64"),
|
||||
mediaType: "application/json",
|
||||
filename: "notes.txt",
|
||||
},
|
||||
limits,
|
||||
});
|
||||
|
||||
expect(result.text).toContain('"revenue"');
|
||||
});
|
||||
});
|
||||
@@ -14,9 +14,12 @@ vi.mock("./media-services.js", () => ({
|
||||
convertHeicToJpeg: (...args: unknown[]) => convertHeicToJpegMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock("@openclaw/media-core/mime", () => ({
|
||||
detectMime: (...args: unknown[]) => detectMimeMock(...args),
|
||||
}));
|
||||
vi.mock("@openclaw/media-core/mime", async () => {
|
||||
const actual = await vi.importActual<typeof import("@openclaw/media-core/mime")>(
|
||||
"@openclaw/media-core/mime",
|
||||
);
|
||||
return { ...actual, detectMime: (...args: unknown[]) => detectMimeMock(...args) };
|
||||
});
|
||||
|
||||
vi.mock("./pdf-extract.js", () => ({
|
||||
extractPdfContent: (...args: unknown[]) => extractPdfContentMock(...args),
|
||||
|
||||
+12
-22
@@ -1,7 +1,11 @@
|
||||
// Input file helpers normalize inline, fetched, and local media inputs.
|
||||
import {
|
||||
classifyAttachmentBytes,
|
||||
type AttachmentClassification,
|
||||
} from "@openclaw/media-core/attachment-classify";
|
||||
import { canonicalizeBase64, estimateBase64DecodedBytes } from "@openclaw/media-core/base64";
|
||||
import { parseMediaContentLength } from "@openclaw/media-core/content-length";
|
||||
import { detectMime } from "@openclaw/media-core/mime";
|
||||
import { detectMime, normalizeMimeType } from "@openclaw/media-core/mime";
|
||||
import {
|
||||
normalizeOptionalLowercaseString,
|
||||
normalizeOptionalString,
|
||||
@@ -155,12 +159,6 @@ function rejectOversizedBase64Payload(params: {
|
||||
}
|
||||
}
|
||||
|
||||
/** Normalizes a MIME value by stripping parameters and lowercasing the media type. */
|
||||
export function normalizeMimeType(value: string | undefined): string | undefined {
|
||||
const [raw] = value?.split(";") ?? [];
|
||||
return normalizeOptionalLowercaseString(raw);
|
||||
}
|
||||
|
||||
/** Parses a Content-Type header into normalized MIME and optional charset values. */
|
||||
function parseContentType(value: string | undefined): {
|
||||
mimeType?: string;
|
||||
@@ -335,20 +333,6 @@ async function normalizeInputImage(params: {
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveInputFileMime(params: {
|
||||
buffer: Buffer;
|
||||
declaredMime?: string;
|
||||
}): Promise<string | undefined> {
|
||||
const sniffedMime = normalizeMimeType(await detectMime({ buffer: params.buffer }));
|
||||
if (!sniffedMime) {
|
||||
return params.declaredMime;
|
||||
}
|
||||
if (sniffedMime === "application/octet-stream") {
|
||||
return params.declaredMime ?? sniffedMime;
|
||||
}
|
||||
return sniffedMime;
|
||||
}
|
||||
|
||||
/** Extracts and normalizes an input_image source from base64 or guarded URL input. */
|
||||
export async function extractImageContentFromSource(
|
||||
source: InputImageSource,
|
||||
@@ -403,6 +387,7 @@ export async function extractFileContentFromSource(params: {
|
||||
source: InputFileSource;
|
||||
limits: InputFileLimits;
|
||||
config?: OpenClawConfig;
|
||||
classification?: AttachmentClassification;
|
||||
}): Promise<InputFileExtractResult> {
|
||||
const { source, limits } = params;
|
||||
const filename = source.filename || "file";
|
||||
@@ -446,7 +431,12 @@ export async function extractFileContentFromSource(params: {
|
||||
throw new Error(`File too large: ${buffer.byteLength} bytes (limit: ${limits.maxBytes} bytes)`);
|
||||
}
|
||||
|
||||
mimeType = await resolveInputFileMime({ buffer, declaredMime: mimeType });
|
||||
// Direct input_file callers declare their content type; the filename is
|
||||
// display metadata and must not override an explicitly allowlisted MIME.
|
||||
const classification =
|
||||
params.classification ?? (await classifyAttachmentBytes({ buffer, declaredMime: mimeType }));
|
||||
mimeType = classification.mime;
|
||||
charset = classification.charset ?? charset;
|
||||
|
||||
if (!mimeType) {
|
||||
throw new Error("input_file missing media type");
|
||||
|
||||
@@ -546,6 +546,7 @@ export const sharedVitestConfig = {
|
||||
},
|
||||
sourcePackageAlias("markdown-core", "code-spans"),
|
||||
sourcePackageAlias("markdown-core", "fences"),
|
||||
sourcePackageAlias("media-core", "attachment-classify"),
|
||||
sourcePackageAlias("media-core", "base64"),
|
||||
sourcePackageAlias("media-core", "constants"),
|
||||
sourcePackageAlias("media-core", "content-length"),
|
||||
|
||||
@@ -108,6 +108,9 @@
|
||||
],
|
||||
"@openclaw/media-generation-core/*": ["./packages/media-generation-core/src/*"],
|
||||
"@openclaw/media-core": ["./packages/media-core/src/index.ts"],
|
||||
"@openclaw/media-core/attachment-classify": [
|
||||
"./packages/media-core/src/attachment-classify.ts"
|
||||
],
|
||||
"@openclaw/media-core/base64": ["./packages/media-core/src/base64.ts"],
|
||||
"@openclaw/media-core/constants": ["./packages/media-core/src/constants.ts"],
|
||||
"@openclaw/media-core/content-length": ["./packages/media-core/src/content-length.ts"],
|
||||
|
||||
Reference in New Issue
Block a user