From e1db0f01fe88c2482c428333512203b94b3f736b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 4 Jun 2026 00:42:21 -0400 Subject: [PATCH] docs: document image pdf model helpers --- src/agents/tools/image-tool.helpers.ts | 13 +++++++++++++ src/agents/tools/pdf-tool.model-config.ts | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/agents/tools/image-tool.helpers.ts b/src/agents/tools/image-tool.helpers.ts index cf3a69381c3f..07b28ea9d705 100644 --- a/src/agents/tools/image-tool.helpers.ts +++ b/src/agents/tools/image-tool.helpers.ts @@ -7,6 +7,12 @@ import { isMinimaxVlmProvider } from "../minimax-vlm.js"; import { findNormalizedProviderValue, normalizeProviderId } from "../model-selection.js"; import { coerceToolModelConfig, type ToolModelConfig } from "./model-config.helpers.js"; +/** + * Shared image-tool helpers for model selection, data URL decoding, and response validation. + * + * These helpers are reused by image, PDF, and media factory paths. + */ +/** Image tool model config uses the shared tool model config shape. */ export type ImageModelConfig = ToolModelConfig; const IMAGE_REASONING_FALLBACK_SIGNATURES = new Set([ @@ -54,6 +60,7 @@ function isImageReasoningFallbackSignature(value: unknown): boolean { return id.startsWith("rs_") && (type === "reasoning" || type.startsWith("reasoning.")); } +/** Detects provider responses that contain only reasoning blocks and no usable image text. */ export function hasImageReasoningOnlyResponse(message: AssistantMessage): boolean { if (extractAssistantText(message).trim() || !Array.isArray(message.content)) { return false; @@ -79,6 +86,7 @@ export function hasImageReasoningOnlyResponse(message: AssistantMessage): boolea return false; } +/** Decodes a base64 image data URL with optional decoded-size protection. */ export function decodeDataUrl( dataUrl: string, opts?: { maxBytes?: number }, @@ -98,6 +106,7 @@ export function decodeDataUrl( } const b64 = (match[2] ?? "").trim(); if (typeof opts?.maxBytes === "number" && estimateBase64DecodedBytes(b64) > opts.maxBytes) { + // Estimate before decoding so oversized inline payloads do not allocate large buffers. throw new Error("Invalid data URL: payload exceeds size limit."); } const buffer = Buffer.from(b64, "base64"); @@ -107,6 +116,7 @@ export function decodeDataUrl( return { buffer, mimeType, kind: "image" }; } +/** Extracts assistant text or throws a provider/model-specific image failure. */ export function coerceImageAssistantText(params: { message: AssistantMessage; provider: string; @@ -131,6 +141,7 @@ export function coerceImageAssistantText(params: { throw new Error(`Image model returned no text (${params.provider}/${params.model}).`); } +/** Reads imageModel defaults from config into the shared tool model config shape. */ export function coerceImageModelConfig(cfg?: OpenClawConfig): ImageModelConfig { return coerceToolModelConfig(cfg?.agents?.defaults?.imageModel); } @@ -211,6 +222,7 @@ function resolveProviderlessConfiguredImageModelRef(params: { ); } +/** Resolves providerless configured image model refs against configured provider models. */ export function resolveConfiguredImageModelRefs(params: { cfg?: OpenClawConfig; imageModelConfig: ImageModelConfig; @@ -235,6 +247,7 @@ export function resolveConfiguredImageModelRefs(params: { }; } +/** Returns the configured vision-capable model for a provider, if present. */ export function resolveProviderVisionModelFromConfig(params: { cfg?: OpenClawConfig; provider: string; diff --git a/src/agents/tools/pdf-tool.model-config.ts b/src/agents/tools/pdf-tool.model-config.ts index d3faba6dad01..9a55363a1cf7 100644 --- a/src/agents/tools/pdf-tool.model-config.ts +++ b/src/agents/tools/pdf-tool.model-config.ts @@ -16,6 +16,12 @@ import { import { hasProviderAuthForTool, resolveDefaultModelRef } from "./model-config.helpers.js"; import { coercePdfModelConfig } from "./pdf-tool.helpers.js"; +/** + * Resolves the model configuration used by the PDF tool. + * + * PDF handling can use explicit PDF config, image-model config, native PDF + * provider support, generic vision models, or document text-extraction fallbacks. + */ function formatProviderModelRef(providerId: string, modelId: string): string { const slash = modelId.indexOf("/"); if (slash > 0 && modelId.slice(0, slash).trim() === providerId) { @@ -57,6 +63,7 @@ function resolveImageCandidateRefs(params: { authStore?: AuthProfileStore; filter?: (providerId: string) => boolean; }): string[] { + // Candidate refs only include providers with usable auth so the tool avoids dead fallbacks. return resolveAutoMediaKeyProviders({ capability: "image", cfg: params.cfg, @@ -193,6 +200,7 @@ export function resolvePdfModelConfigForTool(params: { }): ImageModelConfig | null { const explicitPdf = coercePdfModelConfig(params.cfg); if (explicitPdf.primary?.trim() || (explicitPdf.fallbacks?.length ?? 0) > 0) { + // PDF-specific config wins over generic image model config. return resolveConfiguredImageModelRefs({ cfg: params.cfg, imageModelConfig: explicitPdf, @@ -279,6 +287,7 @@ export function resolvePdfModelConfigForTool(params: { providerOk && textExtractionCandidates.some((ref) => ref.startsWith(`${primary.provider}/`)); if (params.cfg?.models?.providers && typeof params.cfg.models.providers === "object") { + // Configured provider vision models are added even when not present in static media defaults. for (const [providerKey, providerCfg] of Object.entries(params.cfg.models.providers)) { const providerId = providerKey.trim(); const documentImageModel = providerId @@ -327,6 +336,7 @@ export function resolvePdfModelConfigForTool(params: { : [...nativePdfCandidates, ...genericImageCandidates, ...textExtractionCandidates]; if (primary.provider === "google" && googleOk && providerVision && primarySupportsNativePdf) { + // Google native PDF handling is preferred when auth and a configured vision model are present. preferred = providerVision; } else if (providerOk && primarySupportsNativePdf && (providerVision || providerDefault)) { preferred = providerVision ?? `${primary.provider}/${providerDefault}`;