From 8028a3a5b2383fa82179dda7710acb87a46ce13f Mon Sep 17 00:00:00 2001 From: "Jason (Json)" <263060202+fuller-stack-dev@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:27:58 -0600 Subject: [PATCH] feat(anthropic): gate live model discovery on contract coverage (#113757) * feat(anthropic): gate live model discovery on contract coverage Live catalog discovery cloned a template for any newly discovered Claude id, so a future model generation would be selectable while request shaping treated it as pre-4.6 and 400d. Add an opt-in acceptUnknownModel gate to the shared live catalog seam and have the Anthropic plugin accept a discovered model only when Anthropic's advertised capabilities agree with the contracts we would apply. Manifest-published ids bypass the gate and keep their metadata. Co-Authored-By: Claude Fable 5 * docs(plugins): document the live-discovery admission hook Record acceptUnknownModel in the provider-plugin SDK reference: when to use it, that manifest-published ids bypass it, and the fail-closed guidance. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- docs/docs_map.md | 1 + docs/plugins/sdk-provider-plugins.md | 1 + docs/providers/anthropic.md | 14 +++ .../live-model-contract-gate.test.ts | 93 +++++++++++++++++++ .../anthropic/live-model-contract-gate.ts | 64 +++++++++++++ extensions/anthropic/register.runtime.ts | 2 + ...ovider-catalog-live-normalize.gate.test.ts | 54 +++++++++++ ...rovider-catalog-live-normalize.internal.ts | 11 ++- .../provider-catalog-live-runtime.ts | 12 ++- 9 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 extensions/anthropic/live-model-contract-gate.test.ts create mode 100644 extensions/anthropic/live-model-contract-gate.ts create mode 100644 src/plugin-sdk/provider-catalog-live-normalize.gate.test.ts diff --git a/docs/docs_map.md b/docs/docs_map.md index 0d52b17825d6..a5bf010214a0 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -7868,6 +7868,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Usage and cost tracking - H2: Getting started - H2: Claude sessions across computers + - H2: Live model discovery - H2: Thinking defaults (Claude Opus 5, Sonnet 5, Mythos 5, Fable 5, 4.8, and 4.6) - H2: Safety refusal fallback (Claude Opus 5 and Fable 5) - H3: Why this exists diff --git a/docs/plugins/sdk-provider-plugins.md b/docs/plugins/sdk-provider-plugins.md index b17ce2a68920..a3433511e2f0 100644 --- a/docs/plugins/sdk-provider-plugins.md +++ b/docs/plugins/sdk-provider-plugins.md @@ -237,6 +237,7 @@ catalog, API-key auth, and dynamic model resolution. | Network limits | Fetches use OpenClaw's SSRF guard, one 5-second timeout budget across pagination, a 4 MiB response limit per page, and a 50-page limit. Cross-origin pagination links are rejected; credentials are removed after a cross-origin redirect. | | Cache | Successful, non-empty catalogs are cached for 60 seconds by provider, endpoint, and resolved credential. Empty or unusable results are not cached. | | Filtering | Exact live IDs keep their trusted static metadata. New rows are projected conservatively as text/chat models. Disabled, archived, deprecated, explicitly non-chat, embedding, reranking, moderation, speech, image-only, and video-only rows are excluded. Use `readRows` only to select rows from a nonstandard response envelope; provider-specific model semantics still belong in a custom catalog. | + | Admission | Optional. Set `acceptUnknownModel: ({ id, record }) => boolean` when your request shaping is model-version specific, so discovery cannot publish a model you cannot yet build a valid request for. It is called only for IDs your static catalog does not already publish; known IDs bypass it and keep their published metadata. Return `false` to drop the row. Providers that omit it keep the previous behavior unchanged. Prefer comparing the vendor's advertised capabilities against your own contract checks over a hand-maintained model list, and fail closed when the row carries no capability data. | | Failure | Live discovery is advisory. Auth, network, timeout, pagination, parsing, empty-catalog, and filtering failures return the provider-owned static seed instead of removing the provider. | For a non-Bearer or nonstandard list endpoint, pass options instead of diff --git a/docs/providers/anthropic.md b/docs/providers/anthropic.md index 42e881f1176b..6075848c54cc 100644 --- a/docs/providers/anthropic.md +++ b/docs/providers/anthropic.md @@ -281,6 +281,20 @@ remain read-only even on a continuation-enabled node. See [Nodes: Claude sessions and transcripts](/nodes#claude-sessions-and-transcripts) for the node command and security boundary. +## Live model discovery + +With an Anthropic API key configured, OpenClaw refreshes the Claude catalog from +Anthropic's models endpoint, so newly published snapshots of supported model +families appear without an OpenClaw release. Models the shipped catalog already +describes always keep their published metadata and pricing. + +A newly discovered model is only offered when Anthropic's advertised +capabilities match the request shaping OpenClaw would apply to it. A brand-new +model generation therefore stays hidden until OpenClaw adds support for it, +rather than appearing in the picker and failing every request. Discovery is +advisory: without an API key, or if the endpoint is unreachable, the shipped +catalog is used unchanged. + ## Thinking defaults (Claude Opus 5, Sonnet 5, Mythos 5, Fable 5, 4.8, and 4.6) Bare family aliases are rolling: `opus` tracks the current supported Claude diff --git a/extensions/anthropic/live-model-contract-gate.test.ts b/extensions/anthropic/live-model-contract-gate.test.ts new file mode 100644 index 000000000000..bd4c42f84b6f --- /dev/null +++ b/extensions/anthropic/live-model-contract-gate.test.ts @@ -0,0 +1,93 @@ +// Anthropic tests cover the live-discovery contract gate. +import { describe, expect, it } from "vitest"; +import { acceptsAnthropicLiveModelContract } from "./live-model-contract-gate.js"; + +/** Capability tree shaped like Anthropic's `/v1/models` response. */ +function capabilities(params: { + adaptive: boolean; + xhigh: boolean; + max: boolean; +}): Record { + return { + capabilities: { + image_input: { supported: true }, + thinking: { + supported: true, + types: { + enabled: { supported: !params.adaptive }, + adaptive: { supported: params.adaptive }, + }, + }, + effort: { + low: { supported: params.max }, + high: { supported: params.max }, + xhigh: { supported: params.xhigh }, + max: { supported: params.max }, + }, + }, + }; +} + +const MODERN = { adaptive: true, xhigh: true, max: true }; +const LEGACY = { adaptive: false, xhigh: false, max: false }; + +describe("acceptsAnthropicLiveModelContract", () => { + it("accepts current models whose advertised capabilities match our contracts", () => { + // Adaptive + full effort range: what the contracts already shape correctly. + for (const id of ["claude-opus-5", "claude-sonnet-5", "claude-fable-5", "claude-opus-4-8"]) { + expect(acceptsAnthropicLiveModelContract({ id, record: capabilities(MODERN) }), id).toBe( + true, + ); + } + }); + + it("accepts legacy models that correctly advertise no adaptive thinking", () => { + // The gate must not confuse "old model we shape with budget_tokens" for + // "model we do not understand" — both miss the modern predicates. + for (const id of ["claude-haiku-4-5", "claude-sonnet-4-5", "claude-3-opus-20240229"]) { + expect(acceptsAnthropicLiveModelContract({ id, record: capabilities(LEGACY) }), id).toBe( + true, + ); + } + }); + + it("rejects an unreleased model generation our contracts do not recognize", () => { + // The regression this gate exists to prevent: a future Claude family that + // needs adaptive shaping but matches no version predicate, so it would be + // sent manual budget_tokens plus sampling params and 400 on every request. + for (const id of ["claude-opus-6", "claude-sonnet-6", "claude-zephyr-1"]) { + expect(acceptsAnthropicLiveModelContract({ id, record: capabilities(MODERN) }), id).toBe( + false, + ); + } + }); + + it("rejects a known model whose advertised capabilities drift from our contracts", () => { + // If Anthropic changes a shipped model's capabilities, our shaping is stale + // for it; hide it rather than keep sending the old request shape. + expect( + acceptsAnthropicLiveModelContract({ + id: "claude-opus-5", + record: capabilities({ adaptive: false, xhigh: true, max: true }), + }), + ).toBe(false); + expect( + acceptsAnthropicLiveModelContract({ + id: "claude-haiku-4-5", + record: capabilities(MODERN), + }), + ).toBe(false); + }); + + it("fails closed when the capability tree is missing or unreadable", () => { + for (const record of [ + {}, + { capabilities: null }, + { capabilities: "yes" }, + { capabilities: {} }, + { capabilities: { thinking: { types: { adaptive: {} } } } }, + ] as Record[]) { + expect(acceptsAnthropicLiveModelContract({ id: "claude-opus-5", record })).toBe(false); + } + }); +}); diff --git a/extensions/anthropic/live-model-contract-gate.ts b/extensions/anthropic/live-model-contract-gate.ts new file mode 100644 index 000000000000..5f09c96e4d9d --- /dev/null +++ b/extensions/anthropic/live-model-contract-gate.ts @@ -0,0 +1,64 @@ +/** + * Contract gate for live-discovered Anthropic models the manifest does not + * publish. Claude request shaping is selected by model-version predicates, so a + * model those predicates do not recognize is shaped like a pre-4.6 model: + * manual `budget_tokens` thinking plus caller sampling parameters. Current + * Claude models reject both, so surfacing an unrecognized model would offer a + * selectable entry whose every request 400s. + * + * Rather than hand-maintaining a family allowlist that rots on each launch, + * accept a discovered model only when Anthropic's advertised capabilities agree + * with what our contracts would apply. Disagreement means our shaping is stale + * for that model, so it stays hidden until the contracts are updated. + */ +import { + supportsClaudeAdaptiveThinking, + supportsClaudeNativeMaxEffort, + supportsClaudeNativeXhighEffort, +} from "openclaw/plugin-sdk/provider-model-shared"; +import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; + +function readCapabilityFlag(root: unknown, path: readonly string[]): boolean | undefined { + let current: unknown = root; + for (const key of path) { + if (!isRecord(current)) { + return undefined; + } + current = current[key]; + } + return typeof current === "boolean" ? current : undefined; +} + +/** + * Anthropic publishes a capability tree on `/v1/models`. Each entry pairs a + * capability we can read from that tree with the contract predicate that must + * agree, so a mismatch on any axis rejects the model. + */ +const CLAUDE_CONTRACT_CAPABILITY_CHECKS = [ + { + path: ["thinking", "types", "adaptive", "supported"], + matches: supportsClaudeAdaptiveThinking, + }, + { path: ["effort", "xhigh", "supported"], matches: supportsClaudeNativeXhighEffort }, + { path: ["effort", "max", "supported"], matches: supportsClaudeNativeMaxEffort }, +] as const; + +/** + * Return whether a live-discovered Claude model can be shaped by the current + * contracts. Fails closed: a model without a readable capability tree is + * rejected, because we cannot prove our shaping matches it. + */ +export function acceptsAnthropicLiveModelContract(params: { + id: string; + record: Record; +}): boolean { + const capabilities = params.record.capabilities; + if (!isRecord(capabilities)) { + return false; + } + const ref = { id: params.id }; + return CLAUDE_CONTRACT_CAPABILITY_CHECKS.every((check) => { + const advertised = readCapabilityFlag(capabilities, check.path); + return advertised !== undefined && advertised === check.matches(ref); + }); +} diff --git a/extensions/anthropic/register.runtime.ts b/extensions/anthropic/register.runtime.ts index 190b63b71065..9055d9f1c4a8 100644 --- a/extensions/anthropic/register.runtime.ts +++ b/extensions/anthropic/register.runtime.ts @@ -58,6 +58,7 @@ import { applyAnthropicConfigDefaults, normalizeAnthropicProviderConfigForProvider, } from "./config-defaults.js"; +import { acceptsAnthropicLiveModelContract } from "./live-model-contract-gate.js"; import { anthropicMediaUnderstandingProvider } from "./media-understanding-provider.js"; import manifest from "./openclaw.plugin.json" with { type: "json" }; import { resolveClaudeCliSyntheticAuth } from "./provider-discovery.js"; @@ -950,6 +951,7 @@ export function buildAnthropicProvider(): ProviderPlugin { ...(key ? { "x-api-key": key } : {}), }; }, + acceptUnknownModel: acceptsAnthropicLiveModelContract, }, }), }, diff --git a/src/plugin-sdk/provider-catalog-live-normalize.gate.test.ts b/src/plugin-sdk/provider-catalog-live-normalize.gate.test.ts new file mode 100644 index 000000000000..ba80ffe28e2b --- /dev/null +++ b/src/plugin-sdk/provider-catalog-live-normalize.gate.test.ts @@ -0,0 +1,54 @@ +// Plugin SDK tests cover the live-discovery unknown-model gate. +import { describe, expect, it } from "vitest"; +import { buildOpenAICompatibleLiveModels } from "./provider-catalog-live-normalize.internal.js"; +import type { ModelProviderConfig } from "./provider-model-shared.js"; + +const fallback = { + baseUrl: "https://api.example.com", + api: "anthropic-messages", + models: [ + { + id: "known-model", + name: "Known", + reasoning: true, + input: ["text"], + cost: { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }, + contextWindow: 1_000_000, + maxTokens: 128_000, + }, + ], +} as unknown as ModelProviderConfig; + +const rows = [ + { id: "known-model", object: "model" }, + { id: "brand-new-model", object: "model" }, +]; + +describe("live discovery unknown-model gate", () => { + it("publishes discovered models unchanged when no gate is supplied", () => { + const ids = buildOpenAICompatibleLiveModels(rows, fallback).map((m) => m.id); + expect(ids).toContain("known-model"); + expect(ids).toContain("brand-new-model"); + }); + + it("drops only the ids the manifest does not publish when the gate rejects", () => { + const seen: string[] = []; + const models = buildOpenAICompatibleLiveModels(rows, fallback, ({ id }) => { + seen.push(id); + return false; + }); + // The manifest-published row bypasses the gate entirely and survives. + expect(models.map((m) => m.id)).toEqual(["known-model"]); + expect(seen).toEqual(["brand-new-model"]); + }); + + it("keeps the manifest entry verbatim for published ids", () => { + const [model] = buildOpenAICompatibleLiveModels(rows, fallback, () => false); + expect(model).toEqual(fallback.models[0]); + }); + + it("admits unknown ids the gate accepts", () => { + const ids = buildOpenAICompatibleLiveModels(rows, fallback, () => true).map((m) => m.id); + expect(ids).toEqual(["brand-new-model", "known-model"]); + }); +}); diff --git a/src/plugin-sdk/provider-catalog-live-normalize.internal.ts b/src/plugin-sdk/provider-catalog-live-normalize.internal.ts index 84a471f1590a..6b55a0ffdadc 100644 --- a/src/plugin-sdk/provider-catalog-live-normalize.internal.ts +++ b/src/plugin-sdk/provider-catalog-live-normalize.internal.ts @@ -172,6 +172,7 @@ function inferLiveModelReasoning(modelId: string): boolean { function buildOpenAICompatibleLiveModel( row: unknown, fallback: ModelProviderConfig, + acceptUnknownModel?: (params: { id: string; record: Record }) => boolean, ): ModelDefinitionConfig | undefined { const record = readLiveModelCatalogRecord(row); const id = readLiveModelString(record, ["id", "model", "model_name", "modelName"]); @@ -202,6 +203,13 @@ function buildOpenAICompatibleLiveModel( if (exact) { return exact; } + // Manifest-published ids returned above are known-good. Everything past this + // point is a model the manifest has never described, so an opted-in provider + // gate decides whether its request shaping is understood well enough to + // surface it at all. + if (acceptUnknownModel && !acceptUnknownModel({ id, record })) { + return undefined; + } const template = findLiveModelTemplate(id, fallback.models); const inputModalities = readLiveModelStringArray( [record, architecture, capabilities, modelInfo], @@ -276,9 +284,10 @@ function buildOpenAICompatibleLiveModel( export function buildOpenAICompatibleLiveModels( rows: readonly unknown[], fallback: ModelProviderConfig, + acceptUnknownModel?: (params: { id: string; record: Record }) => boolean, ): ModelDefinitionConfig[] { const models = rows - .map((row) => buildOpenAICompatibleLiveModel(row, fallback)) + .map((row) => buildOpenAICompatibleLiveModel(row, fallback, acceptUnknownModel)) .filter((model): model is ModelDefinitionConfig => Boolean(model)); return [...new Map(models.map((model) => [model.id, model])).values()].toSorted((a, b) => a.id.localeCompare(b.id), diff --git a/src/plugin-sdk/provider-catalog-live-runtime.ts b/src/plugin-sdk/provider-catalog-live-runtime.ts index 081cbf1b4459..f4f339c35457 100644 --- a/src/plugin-sdk/provider-catalog-live-runtime.ts +++ b/src/plugin-sdk/provider-catalog-live-runtime.ts @@ -93,6 +93,13 @@ export type OpenAICompatibleModelDiscoveryOptions = { readRows?: FetchLiveProviderModelRowsParams["readRows"]; /** Provider-specific authorization headers for non-Bearer model-list APIs. */ buildRequestHeaders?: FetchLiveProviderModelRowsParams["buildRequestHeaders"]; + /** + * Gate for discovered ids the manifest does not already publish. Providers + * whose request shaping is model-version specific use this to drop models + * they cannot yet shape, so discovery never surfaces a selectable model that + * would build an invalid request. Manifest-published ids bypass it. + */ + acceptUnknownModel?: (params: { id: string; record: Record }) => boolean; }; export type BuildOpenAICompatibleProviderCatalogParams = { @@ -491,6 +498,7 @@ export async function buildOpenAICompatibleLiveModelProviderConfig(params: { ...params.providerConfig, ...(params.apiKey ? { apiKey: params.apiKey } : {}), }; + const acceptUnknownModel = params.modelDiscovery?.acceptUnknownModel; const endpoint = params.modelDiscovery?.endpointUrl ? resolveFixedLiveModelDiscoveryEndpoint(fallback.baseUrl, params.modelDiscovery.endpointUrl) : resolveLiveModelDiscoveryEndpoint( @@ -513,9 +521,9 @@ export async function buildOpenAICompatibleLiveModelProviderConfig(params: { readRows: params.modelDiscovery?.readRows, buildRequestHeaders: params.modelDiscovery?.buildRequestHeaders, shouldCacheRows: (modelRows) => - buildOpenAICompatibleLiveModels(modelRows, fallback).length > 0, + buildOpenAICompatibleLiveModels(modelRows, fallback, acceptUnknownModel).length > 0, }); - const models = buildOpenAICompatibleLiveModels(rows, fallback); + const models = buildOpenAICompatibleLiveModels(rows, fallback, acceptUnknownModel); if (models.length > 0) { return { ...fallback, models }; }