docs: document image pdf model helpers

This commit is contained in:
Peter Steinberger
2026-06-04 00:42:21 -04:00
parent b1053ef9e9
commit e1db0f01fe
2 changed files with 23 additions and 0 deletions
+13
View File
@@ -7,6 +7,12 @@ import { isMinimaxVlmProvider } from "../minimax-vlm.js";
import { findNormalizedProviderValue, normalizeProviderId } from "../model-selection.js";
import { coerceToolModelConfig, type ToolModelConfig } from "./model-config.helpers.js";
/**
* Shared image-tool helpers for model selection, data URL decoding, and response validation.
*
* These helpers are reused by image, PDF, and media factory paths.
*/
/** Image tool model config uses the shared tool model config shape. */
export type ImageModelConfig = ToolModelConfig;
const IMAGE_REASONING_FALLBACK_SIGNATURES = new Set([
@@ -54,6 +60,7 @@ function isImageReasoningFallbackSignature(value: unknown): boolean {
return id.startsWith("rs_") && (type === "reasoning" || type.startsWith("reasoning."));
}
/** Detects provider responses that contain only reasoning blocks and no usable image text. */
export function hasImageReasoningOnlyResponse(message: AssistantMessage): boolean {
if (extractAssistantText(message).trim() || !Array.isArray(message.content)) {
return false;
@@ -79,6 +86,7 @@ export function hasImageReasoningOnlyResponse(message: AssistantMessage): boolea
return false;
}
/** Decodes a base64 image data URL with optional decoded-size protection. */
export function decodeDataUrl(
dataUrl: string,
opts?: { maxBytes?: number },
@@ -98,6 +106,7 @@ export function decodeDataUrl(
}
const b64 = (match[2] ?? "").trim();
if (typeof opts?.maxBytes === "number" && estimateBase64DecodedBytes(b64) > opts.maxBytes) {
// Estimate before decoding so oversized inline payloads do not allocate large buffers.
throw new Error("Invalid data URL: payload exceeds size limit.");
}
const buffer = Buffer.from(b64, "base64");
@@ -107,6 +116,7 @@ export function decodeDataUrl(
return { buffer, mimeType, kind: "image" };
}
/** Extracts assistant text or throws a provider/model-specific image failure. */
export function coerceImageAssistantText(params: {
message: AssistantMessage;
provider: string;
@@ -131,6 +141,7 @@ export function coerceImageAssistantText(params: {
throw new Error(`Image model returned no text (${params.provider}/${params.model}).`);
}
/** Reads imageModel defaults from config into the shared tool model config shape. */
export function coerceImageModelConfig(cfg?: OpenClawConfig): ImageModelConfig {
return coerceToolModelConfig(cfg?.agents?.defaults?.imageModel);
}
@@ -211,6 +222,7 @@ function resolveProviderlessConfiguredImageModelRef(params: {
);
}
/** Resolves providerless configured image model refs against configured provider models. */
export function resolveConfiguredImageModelRefs(params: {
cfg?: OpenClawConfig;
imageModelConfig: ImageModelConfig;
@@ -235,6 +247,7 @@ export function resolveConfiguredImageModelRefs(params: {
};
}
/** Returns the configured vision-capable model for a provider, if present. */
export function resolveProviderVisionModelFromConfig(params: {
cfg?: OpenClawConfig;
provider: string;
+10
View File
@@ -16,6 +16,12 @@ import {
import { hasProviderAuthForTool, resolveDefaultModelRef } from "./model-config.helpers.js";
import { coercePdfModelConfig } from "./pdf-tool.helpers.js";
/**
* Resolves the model configuration used by the PDF tool.
*
* PDF handling can use explicit PDF config, image-model config, native PDF
* provider support, generic vision models, or document text-extraction fallbacks.
*/
function formatProviderModelRef(providerId: string, modelId: string): string {
const slash = modelId.indexOf("/");
if (slash > 0 && modelId.slice(0, slash).trim() === providerId) {
@@ -57,6 +63,7 @@ function resolveImageCandidateRefs(params: {
authStore?: AuthProfileStore;
filter?: (providerId: string) => boolean;
}): string[] {
// Candidate refs only include providers with usable auth so the tool avoids dead fallbacks.
return resolveAutoMediaKeyProviders({
capability: "image",
cfg: params.cfg,
@@ -193,6 +200,7 @@ export function resolvePdfModelConfigForTool(params: {
}): ImageModelConfig | null {
const explicitPdf = coercePdfModelConfig(params.cfg);
if (explicitPdf.primary?.trim() || (explicitPdf.fallbacks?.length ?? 0) > 0) {
// PDF-specific config wins over generic image model config.
return resolveConfiguredImageModelRefs({
cfg: params.cfg,
imageModelConfig: explicitPdf,
@@ -279,6 +287,7 @@ export function resolvePdfModelConfigForTool(params: {
providerOk && textExtractionCandidates.some((ref) => ref.startsWith(`${primary.provider}/`));
if (params.cfg?.models?.providers && typeof params.cfg.models.providers === "object") {
// Configured provider vision models are added even when not present in static media defaults.
for (const [providerKey, providerCfg] of Object.entries(params.cfg.models.providers)) {
const providerId = providerKey.trim();
const documentImageModel = providerId
@@ -327,6 +336,7 @@ export function resolvePdfModelConfigForTool(params: {
: [...nativePdfCandidates, ...genericImageCandidates, ...textExtractionCandidates];
if (primary.provider === "google" && googleOk && providerVision && primarySupportsNativePdf) {
// Google native PDF handling is preferred when auth and a configured vision model are present.
preferred = providerVision;
} else if (providerOk && primarySupportsNativePdf && (providerVision || providerDefault)) {
preferred = providerVision ?? `${primary.provider}/${providerDefault}`;