diff --git a/src/agents/transcript-redact-images.ts b/src/agents/transcript-redact-images.ts new file mode 100644 index 000000000000..68e55312f739 --- /dev/null +++ b/src/agents/transcript-redact-images.ts @@ -0,0 +1,129 @@ +import { + sanitizeInlineImageBase64, + sanitizeInlineImageDataUrlForStorage, +} from "@openclaw/media-core/inline-image-data-url"; + +const isImageMimeType = (value: unknown): value is string => + typeof value === "string" && /^image\//iu.test(value.trim()); + +const normalizeImageMimeType = (value: unknown): string | undefined => + isImageMimeType(value) ? value.trim().toLowerCase() : undefined; + +function imageMimeTypeForRecord(value: Record): string | undefined { + return ( + normalizeImageMimeType(value.mimeType) ?? + normalizeImageMimeType(value.mediaType) ?? + normalizeImageMimeType(value.media_type) + ); +} + +function imageMimeTypeFieldsForRecord(value: Record): string[] { + return ["mimeType", "mediaType", "media_type"].filter((key) => isImageMimeType(value[key])); +} + +function sanitizeOpaqueImageBase64( + base64: string, + mimeType: string | undefined, +): { mimeType: string; base64: string } | undefined { + return mimeType ? sanitizeInlineImageBase64({ mimeType, base64 }) : undefined; +} + +function isValidOpaqueImageBase64(base64: string, mimeType: string | undefined): boolean { + return sanitizeOpaqueImageBase64(base64, mimeType) !== undefined; +} + +function isOpaqueImageDataBlock(value: Record): boolean { + return ( + (value.type === "image" || value.type === "base64") && + typeof value.data === "string" && + isValidOpaqueImageBase64(value.data, imageMimeTypeForRecord(value)) + ); +} + +export function sanitizeTranscriptImageRecord( + source: Record, +): Record | undefined { + const isImageBlock = source.type === "image"; + const isBase64SourceBlock = source.type === "base64"; + if ((!isImageBlock && !isBase64SourceBlock) || typeof source.data !== "string") { + return undefined; + } + const mimeTypeFields = imageMimeTypeFieldsForRecord(source); + if (mimeTypeFields.length === 0) { + return undefined; + } + const sanitized = sanitizeOpaqueImageBase64(source.data, imageMimeTypeForRecord(source)); + if (!sanitized) { + return undefined; + } + const hasCanonicalMimeTypes = mimeTypeFields.every((key) => source[key] === sanitized.mimeType); + if (source.data === sanitized.base64 && hasCanonicalMimeTypes) { + return source; + } + const next: Record = { ...source, data: sanitized.base64 }; + for (const field of mimeTypeFields) { + next[field] = sanitized.mimeType; + } + return next; +} + +function startsWithDataUrl(value: string): boolean { + return value.slice(0, "data:".length).toLowerCase() === "data:"; +} + +function sanitizeImageDataUrlField( + source: Record, + key: string, + value: string, +): string | undefined { + if (!startsWithDataUrl(value)) { + return undefined; + } + const isImageDataUrlField = + (source.type === "input_image" && key === "image_url") || + ((source.type === "image" || source.type === "image_url") && key === "url") || + (source.type === "image" && (key === "source" || key === "data")); + return isImageDataUrlField ? sanitizeInlineImageDataUrlForStorage(value) : undefined; +} + +export function sanitizeTranscriptImageDataUrlField(params: { + source: Record; + key: string; + value: string; + preserveImageDataUrlFields: boolean; +}): string | undefined { + if (params.preserveImageDataUrlFields && params.key === "url") { + return startsWithDataUrl(params.value) + ? sanitizeInlineImageDataUrlForStorage(params.value) + : undefined; + } + return sanitizeImageDataUrlField(params.source, params.key, params.value); +} + +export function shouldPreserveTranscriptImagePayload( + source: Record, + key: string, + item: unknown, + preserveImageDataUrlFields: boolean, +): boolean { + if (typeof item !== "string") { + return false; + } + if (key === "data" && isOpaqueImageDataBlock(source)) { + return true; + } + if (preserveImageDataUrlFields && key === "url") { + return startsWithDataUrl(item) && sanitizeInlineImageDataUrlForStorage(item) !== undefined; + } + return sanitizeImageDataUrlField(source, key, item) !== undefined; +} + +export function shouldPreserveNestedTranscriptImageDataUrlFields( + source: Record, + key: string, +): boolean { + return ( + key === "image_url" && + (source.type === "image_url" || source.type === "input_image" || source.type === "image") + ); +} diff --git a/src/agents/transcript-redact.ts b/src/agents/transcript-redact.ts index c631e37c6cae..0e9a71d27392 100644 --- a/src/agents/transcript-redact.ts +++ b/src/agents/transcript-redact.ts @@ -3,10 +3,6 @@ * * Applies logging redaction rules to persisted messages while preserving unchanged object identity. */ -import { - sanitizeInlineImageBase64, - sanitizeInlineImageDataUrlForStorage, -} from "@openclaw/media-core/inline-image-data-url"; import { findNormalizedProviderValue } from "@openclaw/model-catalog-core/provider-id"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { readLoggingConfig } from "../logging/config.js"; @@ -18,6 +14,12 @@ import { import type { ProviderEndpointClass } from "./provider-attribution.js"; import { resolveProviderEndpoint } from "./provider-attribution.js"; import type { AgentMessage } from "./runtime/index.js"; +import { + sanitizeTranscriptImageDataUrlField, + sanitizeTranscriptImageRecord, + shouldPreserveNestedTranscriptImageDataUrlFields, + shouldPreserveTranscriptImagePayload, +} from "./transcript-redact-images.js"; function resolveTranscriptRedactPatterns(patterns?: string[]) { return patterns && patterns.length > 0 ? [...patterns, ...getDefaultRedactPatterns()] : undefined; @@ -68,126 +70,6 @@ function isPlainTranscriptObject(value: object): value is Record - typeof value === "string" && /^image\//iu.test(value.trim()); - -const normalizeImageMimeType = (value: unknown): string | undefined => - isImageMimeType(value) ? value.trim().toLowerCase() : undefined; - -function imageMimeTypeForRecord(value: Record): string | undefined { - return ( - normalizeImageMimeType(value.mimeType) ?? - normalizeImageMimeType(value.mediaType) ?? - normalizeImageMimeType(value.media_type) - ); -} - -function imageMimeTypeFieldsForRecord(value: Record): string[] { - return ["mimeType", "mediaType", "media_type"].filter((key) => isImageMimeType(value[key])); -} - -function sanitizeOpaqueImageBase64( - base64: string, - mimeType: string | undefined, -): { mimeType: string; base64: string } | undefined { - return mimeType ? sanitizeInlineImageBase64({ mimeType, base64 }) : undefined; -} - -function isValidOpaqueImageBase64(base64: string, mimeType: string | undefined): boolean { - return sanitizeOpaqueImageBase64(base64, mimeType) !== undefined; -} - -function isTranscriptImageContentBlock(value: Record): boolean { - return ( - value.type === "image" && - typeof value.data === "string" && - isValidOpaqueImageBase64(value.data, imageMimeTypeForRecord(value)) - ); -} - -function isImageBase64SourceBlock(value: Record): boolean { - return ( - value.type === "base64" && - typeof value.data === "string" && - isValidOpaqueImageBase64(value.data, imageMimeTypeForRecord(value)) - ); -} - -function sanitizeImageRecord(source: Record): Record | undefined { - const isImageBlock = source.type === "image"; - const isBase64SourceBlock = source.type === "base64"; - if ((!isImageBlock && !isBase64SourceBlock) || typeof source.data !== "string") { - return undefined; - } - const mimeTypeFields = imageMimeTypeFieldsForRecord(source); - if (mimeTypeFields.length === 0) { - return undefined; - } - const sanitized = sanitizeOpaqueImageBase64(source.data, imageMimeTypeForRecord(source)); - if (!sanitized) { - return undefined; - } - const hasCanonicalMimeTypes = mimeTypeFields.every((key) => source[key] === sanitized.mimeType); - if (source.data === sanitized.base64 && hasCanonicalMimeTypes) { - return source; - } - const next: Record = { ...source, data: sanitized.base64 }; - for (const field of mimeTypeFields) { - next[field] = sanitized.mimeType; - } - return next; -} - -function startsWithDataUrl(value: string): boolean { - return value.slice(0, "data:".length).toLowerCase() === "data:"; -} - -function sanitizeImageDataUrlField( - source: Record, - key: string, - value: string, -): string | undefined { - if (!startsWithDataUrl(value)) { - return undefined; - } - const isImageDataUrlField = - (source.type === "input_image" && key === "image_url") || - ((source.type === "image" || source.type === "image_url") && key === "url") || - (source.type === "image" && (key === "source" || key === "data")); - return isImageDataUrlField ? sanitizeInlineImageDataUrlForStorage(value) : undefined; -} - -function shouldPreserveOpaqueImagePayload( - source: Record, - key: string, - item: unknown, - preserveImageDataUrlFields: boolean, -): boolean { - if (typeof item !== "string") { - return false; - } - if ( - key === "data" && - (isTranscriptImageContentBlock(source) || isImageBase64SourceBlock(source)) - ) { - return true; - } - if (preserveImageDataUrlFields && key === "url") { - return startsWithDataUrl(item) && sanitizeInlineImageDataUrlForStorage(item) !== undefined; - } - return sanitizeImageDataUrlField(source, key, item) !== undefined; -} - -function shouldPreserveNestedImageDataUrlFields( - source: Record, - key: string, -): boolean { - return ( - key === "image_url" && - (source.type === "image_url" || source.type === "input_image" || source.type === "image") - ); -} - type TranscriptValueLocation = | "root" | "assistant-content-array" @@ -628,7 +510,7 @@ function redactTranscriptStructuredValue( } seen.add(value); - const sanitizedImageRecord = sanitizeImageRecord(value); + const sanitizedImageRecord = sanitizeTranscriptImageRecord(value); const source = sanitizedImageRecord ?? value; const currentAssistantRoute = location === "root" && source.role === "assistant" @@ -708,12 +590,12 @@ function redactTranscriptStructuredValue( continue; } if (typeof item === "string") { - const sanitizedDataUrl = - preserveImageDataUrlFields && key === "url" - ? startsWithDataUrl(item) - ? sanitizeInlineImageDataUrlForStorage(item) - : undefined - : sanitizeImageDataUrlField(source, key, item); + const sanitizedDataUrl = sanitizeTranscriptImageDataUrlField({ + source, + key, + value: item, + preserveImageDataUrlFields, + }); if (sanitizedDataUrl !== undefined) { if (sanitizedDataUrl !== item) { next ??= { ...source }; @@ -722,7 +604,7 @@ function redactTranscriptStructuredValue( continue; } } - if (shouldPreserveOpaqueImagePayload(source, key, item, preserveImageDataUrlFields)) { + if (shouldPreserveTranscriptImagePayload(source, key, item, preserveImageDataUrlFields)) { continue; } const redacted = redactTranscriptStructuredValue( @@ -730,7 +612,7 @@ function redactTranscriptStructuredValue( cfg, key, seen, - preserveImageDataUrlFields || shouldPreserveNestedImageDataUrlFields(source, key), + preserveImageDataUrlFields || shouldPreserveNestedTranscriptImageDataUrlFields(source, key), location === "root" && source.role === "assistant" && key === "content" && Array.isArray(item) ? "assistant-content-array" : "nested",