From 98cc6df7ff2332f874d55b1fbb66dbeb09609b52 Mon Sep 17 00:00:00 2001 From: Andy Ye <35905412+TurboTheTurtle@users.noreply.github.com> Date: Mon, 18 May 2026 15:16:01 -0700 Subject: [PATCH] fix(anthropic): preserve Claude image capability (#83756) Summary: - The PR adds Anthropic Claude 4.x image-capability normalization for stale text-only resolved model rows, regression tests for provider and fallback model resolution, and a changelog entry. - Reproducibility: yes. for source-level reproduction: current main gates native images on model.input includi ... s text-only. I did not run the command locally because this review was constrained to read-only inspection. Automerge notes: - PR branch already contained follow-up commit before automerge: fix(anthropic): preserve Claude image capability Validation: - ClawSweeper review passed for head 06dd378ea3f66c78edfc237524578331238d71ee. - Required merge gates passed before the squash merge. Prepared head SHA: 06dd378ea3f66c78edfc237524578331238d71ee Review: https://github.com/openclaw/openclaw/pull/83756#issuecomment-4482116499 Co-authored-by: Andy Ye <35905412+TurboTheTurtle@users.noreply.github.com> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: takhoffman Co-authored-by: takhoffman <781889+takhoffman@users.noreply.github.com> --- CHANGELOG.md | 1 + extensions/anthropic/index.test.ts | 22 +++++++ extensions/anthropic/register.runtime.ts | 57 ++++++++++++++++++- .../model.provider-runtime.test-support.ts | 28 +++++++++ src/agents/pi-embedded-runner/model.test.ts | 25 ++++++++ 5 files changed, 130 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a979e988840f..c6fcb1863ef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ Docs: https://docs.openclaw.ai ### Fixes - Agents/subagents: keep collect-mode announce queues batching unresolved-origin items with compatible same-route messages and resume collection after a true cross-channel drain when a later compatible batch remains. Fixes #83577. +- Providers/Anthropic: preserve native image input for current Claude model rows when stale local catalog data marks them text-only. (#83756) Thanks @TurboTheTurtle. - Control UI: render live tool progress from session-scoped `session.tool` Gateway events so externally started runs show their tool cards in the active session. (#83734) Thanks @TurboTheTurtle. - Outbound: resolve send-capable channel plugins from the active runtime registry when the pinned startup registry only has setup metadata. (#83733) Thanks @TurboTheTurtle. - Browser: enforce current-tab URL allowlist checks for `/act` evaluate/batch actions and `/highlight` routes while leaving tab-management actions unblocked. (#78523) diff --git a/extensions/anthropic/index.test.ts b/extensions/anthropic/index.test.ts index dda93bbea110..d110975f414f 100644 --- a/extensions/anthropic/index.test.ts +++ b/extensions/anthropic/index.test.ts @@ -345,6 +345,28 @@ describe("anthropic provider replay hooks", () => { expect(resolved).toBeUndefined(); }); + it("normalizes stale text-only Claude vision rows to image-capable", async () => { + const provider = await registerSingleProviderPlugin(anthropicPlugin); + + const normalized = provider.normalizeResolvedModel?.({ + provider: "anthropic", + modelId: "claude-sonnet-4-5", + model: { + id: "claude-sonnet-4-5", + name: "Claude Sonnet 4.5", + provider: "anthropic", + api: "anthropic-messages", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200_000, + maxTokens: 64_000, + }, + } as never); + + expect(normalized?.input).toEqual(["text", "image"]); + }); + it("normalizes exact claude opus 4.7 variants to 1M context", async () => { const provider = await registerSingleProviderPlugin(anthropicPlugin); diff --git a/extensions/anthropic/register.runtime.ts b/extensions/anthropic/register.runtime.ts index 16099d6544a4..e7d8dc6dffd2 100644 --- a/extensions/anthropic/register.runtime.ts +++ b/extensions/anthropic/register.runtime.ts @@ -63,11 +63,17 @@ const ANTHROPIC_SONNET_46_DOT_MODEL_ID = "claude-sonnet-4.6"; const ANTHROPIC_SONNET_TEMPLATE_MODEL_IDS = ["claude-sonnet-4-5", "claude-sonnet-4.5"] as const; const ANTHROPIC_MODERN_MODEL_PREFIXES = [ "claude-opus-4-7", + "claude-opus-4.7", "claude-opus-4-6", + "claude-opus-4.6", "claude-sonnet-4-6", + "claude-sonnet-4.6", "claude-opus-4-5", + "claude-opus-4.5", "claude-sonnet-4-5", + "claude-sonnet-4.5", "claude-haiku-4-5", + "claude-haiku-4.5", ] as const; const ANTHROPIC_SETUP_TOKEN_NOTE_LINES = [ "Anthropic setup-token auth is supported in OpenClaw.", @@ -370,6 +376,46 @@ function matchesAnthropicModernModel(modelId: string): boolean { return ANTHROPIC_MODERN_MODEL_PREFIXES.some((prefix) => lower.startsWith(prefix)); } +function hasImageInput(input: unknown): boolean { + return Array.isArray(input) && input.includes("image"); +} + +function supportsAnthropicImageInput(modelId: string, modelName?: string): boolean { + return [modelId, modelName] + .filter((value): value is string => typeof value === "string") + .some((candidate) => matchesAnthropicModernModel(candidate)); +} + +function applyAnthropicImageInputCapability(params: { + modelId: string; + model: ProviderRuntimeModel; +}): ProviderRuntimeModel | undefined { + if (hasImageInput(params.model.input)) { + return undefined; + } + if (!supportsAnthropicImageInput(params.modelId, params.model.name)) { + return undefined; + } + return { + ...params.model, + input: ["text", "image"], + }; +} + +function normalizeAnthropicResolvedModel( + ctx: ProviderNormalizeResolvedModelContext, +): ProviderRuntimeModel | undefined { + const imageCapableModel = applyAnthropicImageInputCapability(ctx) ?? ctx.model; + const contextWindowModel = + applyAnthropicOpus47ContextWindow({ + config: ctx.config, + provider: ctx.provider, + modelId: ctx.modelId, + model: imageCapableModel, + }) ?? imageCapableModel; + return contextWindowModel === ctx.model ? undefined : contextWindowModel; +} + function buildAnthropicAuthDoctorHint(params: { config?: ProviderAuthContext["config"]; store: AuthProfileStore; @@ -576,16 +622,21 @@ export function buildAnthropicProvider(): ProviderPlugin { if (!model) { return undefined; } + const imageCapableModel = + applyAnthropicImageInputCapability({ + modelId: ctx.modelId, + model, + }) ?? model; return ( applyAnthropicOpus47ContextWindow({ config: ctx.config, provider: ctx.provider, modelId: ctx.modelId, - model, - }) ?? model + model: imageCapableModel, + }) ?? imageCapableModel ); }, - normalizeResolvedModel: (ctx) => applyAnthropicOpus47ContextWindow(ctx), + normalizeResolvedModel: (ctx) => normalizeAnthropicResolvedModel(ctx), resolveSyntheticAuth: ({ provider }) => normalizeLowercaseStringOrEmpty(provider) === CLAUDE_CLI_BACKEND_ID ? resolveClaudeCliSyntheticAuth() diff --git a/src/agents/pi-embedded-runner/model.provider-runtime.test-support.ts b/src/agents/pi-embedded-runner/model.provider-runtime.test-support.ts index d694a4e921c7..5525294a630b 100644 --- a/src/agents/pi-embedded-runner/model.provider-runtime.test-support.ts +++ b/src/agents/pi-embedded-runner/model.provider-runtime.test-support.ts @@ -14,6 +14,20 @@ const GOOGLE_GEMINI_CLI_BASE_URL = "https://cloudcode-pa.googleapis.com"; const DEFAULT_CONTEXT_WINDOW = 200_000; const DEFAULT_MAX_TOKENS = 8192; const OPENROUTER_FALLBACK_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }; +const ANTHROPIC_VISION_MODEL_PREFIXES = [ + "claude-opus-4-7", + "claude-opus-4.7", + "claude-opus-4-6", + "claude-opus-4.6", + "claude-sonnet-4-6", + "claude-sonnet-4.6", + "claude-opus-4-5", + "claude-opus-4.5", + "claude-sonnet-4-5", + "claude-sonnet-4.5", + "claude-haiku-4-5", + "claude-haiku-4.5", +] as const; type ModelRegistryLike = { find: (provider: string, modelId: string) => unknown; @@ -91,6 +105,20 @@ function normalizeDynamicModel(params: { provider: string; model: ResolvedModelL } return undefined; } + if (params.provider === "anthropic" || params.provider === "claude-cli") { + const candidates = [params.model.id, params.model.name] + .filter((value): value is string => typeof value === "string") + .map((value) => lowercasePreservingWhitespace(value)) + .filter(Boolean); + const isKnownVisionModel = candidates.some((candidate) => + ANTHROPIC_VISION_MODEL_PREFIXES.some((prefix) => candidate.startsWith(prefix)), + ); + const hasImageInput = Array.isArray(params.model.input) && params.model.input.includes("image"); + if (isKnownVisionModel && !hasImageInput) { + return { ...params.model, input: ["text", "image"] }; + } + return undefined; + } if (params.provider !== "openai-codex") { return undefined; } diff --git a/src/agents/pi-embedded-runner/model.test.ts b/src/agents/pi-embedded-runner/model.test.ts index 6d8c05226489..da9fd0a0e5c9 100644 --- a/src/agents/pi-embedded-runner/model.test.ts +++ b/src/agents/pi-embedded-runner/model.test.ts @@ -1533,6 +1533,31 @@ describe("resolveModel", () => { expect(result.model?.input).toEqual(["text", "image"]); }); + it("repairs stale text-only Anthropic fallback rows for Claude vision models", () => { + const cfg = { + models: { + providers: { + anthropic: { + baseUrl: "https://api.anthropic.com", + api: "anthropic-messages", + models: [ + { + ...makeModel("claude-sonnet-4-5"), + name: "claude-sonnet-4-5", + api: "anthropic-messages", + input: ["text"], + }, + ], + }, + }, + }, + } as unknown as OpenClawConfig; + + const result = resolveModelForTest("anthropic", "claude-sonnet-4-5", "/tmp/agent", cfg); + + expect(result.model?.input).toEqual(["text", "image"]); + }); + it("repairs stale text-only Foundry discovered rows for GPT-family models", () => { const cfg = { models: {