mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-24 19:35:28 -06:00
docs: document image pdf model helpers
This commit is contained in:
@@ -7,6 +7,12 @@ import { isMinimaxVlmProvider } from "../minimax-vlm.js";
|
||||
import { findNormalizedProviderValue, normalizeProviderId } from "../model-selection.js";
|
||||
import { coerceToolModelConfig, type ToolModelConfig } from "./model-config.helpers.js";
|
||||
|
||||
/**
|
||||
* Shared image-tool helpers for model selection, data URL decoding, and response validation.
|
||||
*
|
||||
* These helpers are reused by image, PDF, and media factory paths.
|
||||
*/
|
||||
/** Image tool model config uses the shared tool model config shape. */
|
||||
export type ImageModelConfig = ToolModelConfig;
|
||||
|
||||
const IMAGE_REASONING_FALLBACK_SIGNATURES = new Set([
|
||||
@@ -54,6 +60,7 @@ function isImageReasoningFallbackSignature(value: unknown): boolean {
|
||||
return id.startsWith("rs_") && (type === "reasoning" || type.startsWith("reasoning."));
|
||||
}
|
||||
|
||||
/** Detects provider responses that contain only reasoning blocks and no usable image text. */
|
||||
export function hasImageReasoningOnlyResponse(message: AssistantMessage): boolean {
|
||||
if (extractAssistantText(message).trim() || !Array.isArray(message.content)) {
|
||||
return false;
|
||||
@@ -79,6 +86,7 @@ export function hasImageReasoningOnlyResponse(message: AssistantMessage): boolea
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Decodes a base64 image data URL with optional decoded-size protection. */
|
||||
export function decodeDataUrl(
|
||||
dataUrl: string,
|
||||
opts?: { maxBytes?: number },
|
||||
@@ -98,6 +106,7 @@ export function decodeDataUrl(
|
||||
}
|
||||
const b64 = (match[2] ?? "").trim();
|
||||
if (typeof opts?.maxBytes === "number" && estimateBase64DecodedBytes(b64) > opts.maxBytes) {
|
||||
// Estimate before decoding so oversized inline payloads do not allocate large buffers.
|
||||
throw new Error("Invalid data URL: payload exceeds size limit.");
|
||||
}
|
||||
const buffer = Buffer.from(b64, "base64");
|
||||
@@ -107,6 +116,7 @@ export function decodeDataUrl(
|
||||
return { buffer, mimeType, kind: "image" };
|
||||
}
|
||||
|
||||
/** Extracts assistant text or throws a provider/model-specific image failure. */
|
||||
export function coerceImageAssistantText(params: {
|
||||
message: AssistantMessage;
|
||||
provider: string;
|
||||
@@ -131,6 +141,7 @@ export function coerceImageAssistantText(params: {
|
||||
throw new Error(`Image model returned no text (${params.provider}/${params.model}).`);
|
||||
}
|
||||
|
||||
/** Reads imageModel defaults from config into the shared tool model config shape. */
|
||||
export function coerceImageModelConfig(cfg?: OpenClawConfig): ImageModelConfig {
|
||||
return coerceToolModelConfig(cfg?.agents?.defaults?.imageModel);
|
||||
}
|
||||
@@ -211,6 +222,7 @@ function resolveProviderlessConfiguredImageModelRef(params: {
|
||||
);
|
||||
}
|
||||
|
||||
/** Resolves providerless configured image model refs against configured provider models. */
|
||||
export function resolveConfiguredImageModelRefs(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
imageModelConfig: ImageModelConfig;
|
||||
@@ -235,6 +247,7 @@ export function resolveConfiguredImageModelRefs(params: {
|
||||
};
|
||||
}
|
||||
|
||||
/** Returns the configured vision-capable model for a provider, if present. */
|
||||
export function resolveProviderVisionModelFromConfig(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
provider: string;
|
||||
|
||||
@@ -16,6 +16,12 @@ import {
|
||||
import { hasProviderAuthForTool, resolveDefaultModelRef } from "./model-config.helpers.js";
|
||||
import { coercePdfModelConfig } from "./pdf-tool.helpers.js";
|
||||
|
||||
/**
|
||||
* Resolves the model configuration used by the PDF tool.
|
||||
*
|
||||
* PDF handling can use explicit PDF config, image-model config, native PDF
|
||||
* provider support, generic vision models, or document text-extraction fallbacks.
|
||||
*/
|
||||
function formatProviderModelRef(providerId: string, modelId: string): string {
|
||||
const slash = modelId.indexOf("/");
|
||||
if (slash > 0 && modelId.slice(0, slash).trim() === providerId) {
|
||||
@@ -57,6 +63,7 @@ function resolveImageCandidateRefs(params: {
|
||||
authStore?: AuthProfileStore;
|
||||
filter?: (providerId: string) => boolean;
|
||||
}): string[] {
|
||||
// Candidate refs only include providers with usable auth so the tool avoids dead fallbacks.
|
||||
return resolveAutoMediaKeyProviders({
|
||||
capability: "image",
|
||||
cfg: params.cfg,
|
||||
@@ -193,6 +200,7 @@ export function resolvePdfModelConfigForTool(params: {
|
||||
}): ImageModelConfig | null {
|
||||
const explicitPdf = coercePdfModelConfig(params.cfg);
|
||||
if (explicitPdf.primary?.trim() || (explicitPdf.fallbacks?.length ?? 0) > 0) {
|
||||
// PDF-specific config wins over generic image model config.
|
||||
return resolveConfiguredImageModelRefs({
|
||||
cfg: params.cfg,
|
||||
imageModelConfig: explicitPdf,
|
||||
@@ -279,6 +287,7 @@ export function resolvePdfModelConfigForTool(params: {
|
||||
providerOk && textExtractionCandidates.some((ref) => ref.startsWith(`${primary.provider}/`));
|
||||
|
||||
if (params.cfg?.models?.providers && typeof params.cfg.models.providers === "object") {
|
||||
// Configured provider vision models are added even when not present in static media defaults.
|
||||
for (const [providerKey, providerCfg] of Object.entries(params.cfg.models.providers)) {
|
||||
const providerId = providerKey.trim();
|
||||
const documentImageModel = providerId
|
||||
@@ -327,6 +336,7 @@ export function resolvePdfModelConfigForTool(params: {
|
||||
: [...nativePdfCandidates, ...genericImageCandidates, ...textExtractionCandidates];
|
||||
|
||||
if (primary.provider === "google" && googleOk && providerVision && primarySupportsNativePdf) {
|
||||
// Google native PDF handling is preferred when auth and a configured vision model are present.
|
||||
preferred = providerVision;
|
||||
} else if (providerOk && primarySupportsNativePdf && (providerVision || providerDefault)) {
|
||||
preferred = providerVision ?? `${primary.provider}/${providerDefault}`;
|
||||
|
||||
Reference in New Issue
Block a user