From 31865d3e330692a0fadd1a8c65c935829ce229bf Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Wed, 12 Aug 2026 18:02:56 -0700 Subject: [PATCH] fix(models): bind discovered auth profile at runtime --- .../run/auth-plan.auth-pin.test.ts | 54 +++++++++++++++++++ .../run/auth-plan.test-support.ts | 11 ++++ .../embedded-agent-runner/run/auth-plan.ts | 47 +++++++++++++--- .../prepared-model-runtime-full-catalog.ts | 30 +++++++++++ src/agents/prepared-model-runtime.build.ts | 10 ++-- ...pared-model-runtime.startup-static.test.ts | 3 ++ .../models-list-provider-outcomes.ts | 18 +++---- src/plugins/provider-catalog-outcome.ts | 20 +++++++ ui/src/pages/chat/chat-state-refresh.ts | 23 +++++--- ui/src/pages/chat/chat-state.test.ts | 29 ++++++++-- 10 files changed, 215 insertions(+), 30 deletions(-) create mode 100644 src/agents/prepared-model-runtime-full-catalog.ts diff --git a/src/agents/embedded-agent-runner/run/auth-plan.auth-pin.test.ts b/src/agents/embedded-agent-runner/run/auth-plan.auth-pin.test.ts index ac1b6c0344cd..5296955eee79 100644 --- a/src/agents/embedded-agent-runner/run/auth-plan.auth-pin.test.ts +++ b/src/agents/embedded-agent-runner/run/auth-plan.auth-pin.test.ts @@ -76,4 +76,58 @@ describe("embedded run auth plan provider pin", () => { }, }); }); + + it("prefers the profile whose loaded catalog authorized the selected model", () => { + const preparedModelRuntime = { + modelCatalog: { + entries: [], + routeVariants: [], + providerOutcomes: [ + { + provider: "openai", + profileId: "openai:first", + status: "ready" as const, + modelIds: ["gpt-5.6-sol"], + }, + { + provider: "openai", + profileId: "openai:second", + status: "ready" as const, + modelIds: ["gpt-5.6-terra"], + }, + ], + }, + } as unknown as Parameters< + typeof authPlanTesting.resolveEmbeddedRunPreferredProfileId + >[0]["preparedModelRuntime"]; + + expect( + authPlanTesting.resolveEmbeddedRunPreferredProfileId({ + provider: "openai", + modelId: "gpt-5.6-terra", + preparedModelRuntime, + requestedProfileId: "openai:first", + ignoreAutoPreferredProfile: false, + }), + ).toBe("openai:second"); + expect( + authPlanTesting.resolveEmbeddedRunPreferredProfileId({ + provider: "openai", + modelId: "gpt-5.6-terra", + preparedModelRuntime, + requestedProfileId: "openai:first", + lockedProfileId: "openai:first", + ignoreAutoPreferredProfile: false, + }), + ).toBe("openai:first"); + expect( + authPlanTesting.resolveEmbeddedRunPreferredProfileId({ + provider: "openai", + modelId: "gpt-5.6-sol", + preparedModelRuntime, + requestedProfileId: "openai:first", + ignoreAutoPreferredProfile: false, + }), + ).toBe("openai:first"); + }); }); diff --git a/src/agents/embedded-agent-runner/run/auth-plan.test-support.ts b/src/agents/embedded-agent-runner/run/auth-plan.test-support.ts index 76e5b91c35b5..7a30f8211d41 100644 --- a/src/agents/embedded-agent-runner/run/auth-plan.test-support.ts +++ b/src/agents/embedded-agent-runner/run/auth-plan.test-support.ts @@ -1,5 +1,6 @@ import "./auth-plan.js"; import type { AuthProfileStore } from "../../auth-profiles.js"; +import type { PreparedModelRuntimeSnapshot } from "../../prepared-model-runtime.js"; import type { RunEmbeddedAgentParams } from "./params.js"; type EmbeddedRunAuthPlanTestApi = { @@ -8,6 +9,14 @@ type EmbeddedRunAuthPlanTestApi = { config: RunEmbeddedAgentParams["config"]; externalCliProviderIds: Iterable; }): AuthProfileStore; + resolveEmbeddedRunPreferredProfileId(params: { + provider: string; + modelId: string; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; + requestedProfileId?: string; + lockedProfileId?: string; + ignoreAutoPreferredProfile: boolean; + }): string | undefined; }; function getTestApi(): EmbeddedRunAuthPlanTestApi { @@ -18,4 +27,6 @@ function getTestApi(): EmbeddedRunAuthPlanTestApi { export const testing: EmbeddedRunAuthPlanTestApi = { loadEmbeddedRunAuthProfileStore: (params) => getTestApi().loadEmbeddedRunAuthProfileStore(params), + resolveEmbeddedRunPreferredProfileId: (params) => + getTestApi().resolveEmbeddedRunPreferredProfileId(params), }; diff --git a/src/agents/embedded-agent-runner/run/auth-plan.ts b/src/agents/embedded-agent-runner/run/auth-plan.ts index ac1e20540d3c..feac5222e6e8 100644 --- a/src/agents/embedded-agent-runner/run/auth-plan.ts +++ b/src/agents/embedded-agent-runner/run/auth-plan.ts @@ -1,3 +1,4 @@ +import { listProviderModelAuthorizingProfileIds } from "../../../plugins/provider-catalog-outcome.js"; import { resolveProviderAuthProfileId } from "../../../plugins/provider-runtime.js"; import type { AuthProfileStore } from "../../auth-profiles.js"; import { resolveExternalCliAuthOverlayScopeFromSelection } from "../../auth-profiles/external-cli-auth-selection.js"; @@ -7,6 +8,7 @@ import { ensureAuthProfileStoreWithoutExternalProfiles, } from "../../model-auth.js"; import { OPENAI_PROVIDER_ID } from "../../openai-routing.js"; +import { getLoadedFullModelCatalog } from "../../prepared-model-runtime-full-catalog.js"; import type { PreparedModelRuntimeSnapshot } from "../../prepared-model-runtime.js"; import { createPreparedRuntimeModelMaterializer, @@ -22,6 +24,35 @@ import type { RunEmbeddedAgentParams } from "./params.js"; type ModelResolution = Awaited>; type RuntimeModel = NonNullable; +function resolveEmbeddedRunPreferredProfileId(params: { + provider: string; + modelId: string; + preparedModelRuntime?: PreparedModelRuntimeSnapshot; + requestedProfileId?: string; + lockedProfileId?: string; + ignoreAutoPreferredProfile: boolean; +}): string | undefined { + const loadedCatalog = getLoadedFullModelCatalog(params.preparedModelRuntime); + const authorizingProfileIds = params.lockedProfileId + ? [] + : listProviderModelAuthorizingProfileIds({ + outcomes: + loadedCatalog?.providerOutcomes ?? + params.preparedModelRuntime?.modelCatalog.providerOutcomes, + provider: params.provider, + modelId: params.modelId, + }); + // Discovery made this model selectable with one of these exact profiles. Preserve an already + // authorizing auto-selection; otherwise bind the first attempt to catalog provenance. + const catalogAuthorizedProfileId = + params.requestedProfileId && authorizingProfileIds.includes(params.requestedProfileId) + ? params.requestedProfileId + : authorizingProfileIds[0]; + return params.ignoreAutoPreferredProfile && !params.lockedProfileId + ? undefined + : (params.lockedProfileId ?? catalogAuthorizedProfileId ?? params.requestedProfileId); +} + function loadEmbeddedRunAuthProfileStore(params: { agentDir: string; config: RunEmbeddedAgentParams["config"]; @@ -40,7 +71,7 @@ function loadEmbeddedRunAuthProfileStore(params: { // must stay provable without composing a full embedded runner. if (process.env.VITEST || process.env.NODE_ENV === "test") { (globalThis as Record)[Symbol.for("openclaw.embeddedRunAuthPlanTestApi")] = - { loadEmbeddedRunAuthProfileStore }; + { loadEmbeddedRunAuthProfileStore, resolveEmbeddedRunPreferredProfileId }; } export async function prepareEmbeddedRunAuthPlan(params: { @@ -132,10 +163,14 @@ export async function prepareEmbeddedRunAuthPlan(params: { const requestedProfileId = runParams.authProfileId?.trim() || undefined; const lockedProfileId = runParams.authProfileIdSource === "user" ? requestedProfileId : undefined; - const preferredProfileId = - externalCliAuthScope.ignoreAutoPreferredProfile && !lockedProfileId - ? undefined - : requestedProfileId; + const preferredProfileId = resolveEmbeddedRunPreferredProfileId({ + provider: params.provider, + modelId: params.modelId, + preparedModelRuntime: params.preparedModelRuntime, + requestedProfileId, + lockedProfileId, + ignoreAutoPreferredProfile: externalCliAuthScope.ignoreAutoPreferredProfile, + }); const createAuthPreparation = () => { const harness = params.getAgentHarness(); return prepareAgentRuntimeAuth({ @@ -180,7 +215,7 @@ export async function prepareEmbeddedRunAuthPlan(params: { config: runParams.config, getModel: params.getRuntimeModel, nativeModelOwned: params.nativeModelOwned, - requestedProfileId: runParams.authProfileId, + requestedProfileId: preferredProfileId, providerUsesProfileScopedModelMetadata, resolveModel: ({ config, authProfileId, authProfileMode }) => resolveModelAsync(params.provider, params.modelId, params.agentDir, config, { diff --git a/src/agents/prepared-model-runtime-full-catalog.ts b/src/agents/prepared-model-runtime-full-catalog.ts new file mode 100644 index 000000000000..9f784df7f488 --- /dev/null +++ b/src/agents/prepared-model-runtime-full-catalog.ts @@ -0,0 +1,30 @@ +import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; +import type { PreparedModelRuntimeSnapshot } from "./prepared-model-runtime.types.js"; + +const loadedFullModelCatalogAccessor = Symbol("openclaw.loadedFullModelCatalogAccessor"); + +type SnapshotWithLoadedFullModelCatalogAccessor = PreparedModelRuntimeSnapshot & { + [loadedFullModelCatalogAccessor]?: () => ModelCatalogSnapshot | undefined; +}; + +/** Carries the lifecycle-owned full-catalog closure without widening the public snapshot shape. */ +export function attachLoadedFullModelCatalogAccessor( + snapshot: PreparedModelRuntimeSnapshot, + accessor: () => ModelCatalogSnapshot | undefined, +): void { + Object.defineProperty(snapshot, loadedFullModelCatalogAccessor, { + configurable: false, + enumerable: true, + value: accessor, + writable: false, + }); +} + +/** Returns the full catalog only after this exact generation has already loaded it. */ +export function getLoadedFullModelCatalog( + snapshot: PreparedModelRuntimeSnapshot | undefined, +): ModelCatalogSnapshot | undefined { + return (snapshot as SnapshotWithLoadedFullModelCatalogAccessor | undefined)?.[ + loadedFullModelCatalogAccessor + ]?.(); +} diff --git a/src/agents/prepared-model-runtime.build.ts b/src/agents/prepared-model-runtime.build.ts index 5e6d2b3baece..0c2d6b75446a 100644 --- a/src/agents/prepared-model-runtime.build.ts +++ b/src/agents/prepared-model-runtime.build.ts @@ -7,6 +7,7 @@ import { resolveUsableAgentCredentialModes } from "./agent-auth-credentials.js"; import { getPreparedRuntimeAuthMaterializations } from "./auth-profiles/runtime-materializations.js"; import type { ModelCatalogSnapshot } from "./model-catalog.types.js"; import { setPreparedModelRuntimeAuthMaterializations } from "./prepared-model-runtime-auth.js"; +import { attachLoadedFullModelCatalogAccessor } from "./prepared-model-runtime-full-catalog.js"; import { PreparedModelRuntimePublicationSupersededError } from "./prepared-model-runtime.errors.js"; import { fingerprintPreparedRuntimeFacts, @@ -37,6 +38,7 @@ const MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS = 1; const limitFullModelCatalogBuild = pLimit(MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS); type PreparedModelRuntimeCatalogAccess = Readonly<{ + getLoadedFullModelCatalog: () => ModelCatalogSnapshot | undefined; loadFullModelCatalog: () => Promise; }>; type PreparedModelRuntimeBuildGuards = @@ -117,6 +119,7 @@ function createFullModelCatalogAccess(params: { } }; return { + getLoadedFullModelCatalog: () => fullCatalog, loadFullModelCatalog: () => { if (fullCatalog) { return Promise.resolve(fullCatalog); @@ -177,7 +180,7 @@ function createSnapshot( const authStorage = AuthStorage.inMemory(credentials); return { authStorage, modelRegistry: templateModelRegistry.fork(authStorage) }; }; - const snapshot: PreparedModelRuntimeSnapshot = Object.freeze({ + const snapshot: PreparedModelRuntimeSnapshot = { ...(input.agentId ? { agentId: input.agentId } : {}), agentDir: input.agentDir, activeProjectKeys: [], @@ -195,12 +198,13 @@ function createSnapshot( configuredRuntimeModels, inlineProviderModels, createStores, - }); + }; + attachLoadedFullModelCatalogAccessor(snapshot, catalogAccess.getLoadedFullModelCatalog); setPreparedModelRuntimeAuthMaterializations( snapshot, Object.freeze([...getPreparedRuntimeAuthMaterializations(input.agentDir)]), ); - return snapshot; + return Object.freeze(snapshot); } async function buildSnapshotBatch( diff --git a/src/agents/prepared-model-runtime.startup-static.test.ts b/src/agents/prepared-model-runtime.startup-static.test.ts index 01fd2e07eeba..96f61b5d0cfe 100644 --- a/src/agents/prepared-model-runtime.startup-static.test.ts +++ b/src/agents/prepared-model-runtime.startup-static.test.ts @@ -178,6 +178,7 @@ vi.mock("../logging/subsystem.js", () => ({ const { getPreparedModelRuntimeSnapshot, refreshPreparedModelRuntimeSnapshots } = await import("./prepared-model-runtime.js"); +const { getLoadedFullModelCatalog } = await import("./prepared-model-runtime-full-catalog.js"); const { prepareScopedReadOnlyLiveModelCatalog, prepareScopedReadOnlyModelCatalog } = await import("./prepared-model-runtime.scoped-catalog.js"); const { resetPreparedModelRuntimeSnapshotsForTest } = @@ -380,7 +381,9 @@ describe("prepared model runtime Gateway catalog mode", () => { expect(snapshot?.pluginRegistry).toBeDefined(); expect(snapshot?.messageToolCatalog).toBeUndefined(); expect(snapshot?.mediaCapabilityProviders).toBeDefined(); + expect(getLoadedFullModelCatalog(snapshot)).toBeUndefined(); const fullCatalog = await snapshot?.loadFullModelCatalog?.(); + expect(getLoadedFullModelCatalog(snapshot)).toBe(fullCatalog); expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); expect(mocks.planOpenClawModelsJsonSource).toHaveBeenCalledWith( config, diff --git a/src/gateway/server-methods/models-list-provider-outcomes.ts b/src/gateway/server-methods/models-list-provider-outcomes.ts index 79a68062367e..d34914e72391 100644 --- a/src/gateway/server-methods/models-list-provider-outcomes.ts +++ b/src/gateway/server-methods/models-list-provider-outcomes.ts @@ -1,6 +1,9 @@ import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; import type { ModelAuthAvailabilityEvaluation } from "../../agents/model-auth-availability.js"; -import type { ProviderCatalogOutcome } from "../../plugins/provider-catalog.types.js"; +import { + listProviderModelAuthorizingProfileIds, + type ProviderCatalogOutcome, +} from "../../plugins/provider-catalog-outcome.js"; export function projectPublicProviderCatalogOutcomes( outcomes: readonly ProviderCatalogOutcome[] | undefined, @@ -22,14 +25,11 @@ export function applyProviderCatalogOutcomesToModelAuth(params: { const provider = normalizeProviderId(params.provider); const outcomes = params.outcomes?.filter((outcome) => normalizeProviderId(outcome.provider) === provider) ?? []; - const modelId = params.modelId.trim().toLowerCase(); - const authorizingProfileIds = outcomes.flatMap((outcome) => - outcome.status === "ready" && - outcome.profileId && - outcome.modelIds?.some((candidate) => candidate.trim().toLowerCase() === modelId) - ? [outcome.profileId] - : [], - ); + const authorizingProfileIds = listProviderModelAuthorizingProfileIds({ + outcomes, + provider, + modelId: params.modelId, + }); if ( authorizingProfileIds.length > 0 && !authorizingProfileIds.includes(params.resolved.selectedProfileId ?? "") diff --git a/src/plugins/provider-catalog-outcome.ts b/src/plugins/provider-catalog-outcome.ts index 89110dad5245..a9a8c5623d78 100644 --- a/src/plugins/provider-catalog-outcome.ts +++ b/src/plugins/provider-catalog-outcome.ts @@ -1,3 +1,5 @@ +import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; + export type ProviderCatalogOutcome = { provider: string; /** Auth profile tested by discovery; omission means provider-wide auth. */ @@ -6,3 +8,21 @@ export type ProviderCatalogOutcome = { /** Models the tested profile's authoritative catalog allowed in this generation. */ modelIds?: readonly string[]; }; + +/** Profiles whose authoritative live catalog included this exact model. */ +export function listProviderModelAuthorizingProfileIds(params: { + outcomes: readonly ProviderCatalogOutcome[] | undefined; + provider: string; + modelId: string; +}): string[] { + const provider = normalizeProviderId(params.provider); + const modelId = params.modelId.trim().toLowerCase(); + return (params.outcomes ?? []).flatMap((outcome) => + normalizeProviderId(outcome.provider) === provider && + outcome.status === "ready" && + outcome.profileId && + outcome.modelIds?.some((candidate) => candidate.trim().toLowerCase() === modelId) + ? [outcome.profileId] + : [], + ); +} diff --git a/ui/src/pages/chat/chat-state-refresh.ts b/ui/src/pages/chat/chat-state-refresh.ts index 71753c9881a8..d2af21203690 100644 --- a/ui/src/pages/chat/chat-state-refresh.ts +++ b/ui/src/pages/chat/chat-state-refresh.ts @@ -227,7 +227,9 @@ async function refreshMissingChatMetadata( const commandsRefresh = applied.commands ? Promise.resolve() : refreshCompatibilityCommands(request); - const preserveModels = opts?.preserveModelCatalogOnFallback; + const preserveModels = + opts?.preserveModelCatalogOnFallback || + isGatewayMethodAdvertised(request.host as unknown as ChatState, "models.list") === false; const modelsRefresh = applied.models || preserveModels ? Promise.resolve() @@ -267,11 +269,12 @@ export async function refreshChatMetadata( if (!ownsChatMetadataRequest(request)) { return EMPTY_CHAT_METADATA_APPLY_RESULT; } - // chat.metadata remains the compatibility source for commands only. Picker inventory must - // always come from the live, agent-scoped models.list result so stale static models cannot - // reappear when chat.startup omits metadata or an older Gateway serves this fallback path. + // Current Gateways use the live, agent-scoped catalog. A Gateway that explicitly lacks + // models.list keeps its metadata catalog so the compatibility path does not empty the picker. + const useMetadataModels = + isGatewayMethodAdvertised(host as unknown as ChatState, "models.list") === false; const metadataApplied = applyChatMetadataResult(host, client, agentId, result, { - models: false, + models: useMetadataModels, }); if (!metadataApplied.models || !metadataApplied.commands) { await refreshMissingChatMetadata(request, metadataApplied, opts); @@ -461,9 +464,13 @@ export function refreshPageChat(host: ChatPageHost, opts?: ChatRefreshOptions) { return; } rememberChatMetadata(client, agentId, metadata); - // Startup metadata stays on the published static catalog so opening chat never waits on - // provider discovery. The explicit models.list read below owns the live picker inventory. - const applied = applyChatMetadataResult(host, client, agentId, metadata, { models: false }); + // Only an explicitly legacy Gateway owns picker inventory through startup metadata. + // Current and capability-unknown Gateways continue into live, agent-scoped discovery. + const useMetadataModels = + isGatewayMethodAdvertised(host as unknown as ChatState, "models.list") === false; + const applied = applyChatMetadataResult(host, client, agentId, metadata, { + models: useMetadataModels, + }); if (!applied.models || !applied.commands) { await refreshMissingChatMetadata(request, applied, { refreshModelCatalog: true }); } diff --git a/ui/src/pages/chat/chat-state.test.ts b/ui/src/pages/chat/chat-state.test.ts index c5b89a56a23e..44b01267a4d4 100644 --- a/ui/src/pages/chat/chat-state.test.ts +++ b/ui/src/pages/chat/chat-state.test.ts @@ -1410,7 +1410,7 @@ describe("refreshChatMetadata", () => { agentsList: null, assistantAgentId: "main", client: { request }, - hello: { features: { methods: ["chat.metadata"] } }, + hello: { features: { methods: ["chat.metadata", "models.list"] } }, sessionKey: "agent:work:main", ...overrides, } as unknown as ChatPageHost; @@ -1644,7 +1644,7 @@ describe("refreshChatMetadata", () => { chatMetadataRequestVersion: 2, chatModelCatalog: [{ id: "stale-model", name: "Stale Model", provider: "openai" }], chatModelsLoading: true, - hello: { features: { methods: [] } }, + hello: { features: { methods: ["models.list"] } }, sessionKey: "agent:main:main", }); @@ -1658,6 +1658,27 @@ describe("refreshChatMetadata", () => { expect(request).toHaveBeenCalledTimes(2); }); + it("retains metadata models when the gateway explicitly lacks models.list", async () => { + const metadataModel = { + id: "metadata-model", + name: "Metadata Model", + provider: "openai", + available: true, + }; + const request = vi.fn(async (method: string) => { + expect(method).toBe("chat.metadata"); + return { commands: [], models: [metadataModel] }; + }); + const state = createMetadataState(request, { + hello: { features: { methods: ["chat.metadata"] } }, + }); + + await refreshChatMetadata(state); + + expect(state.chatModelCatalog).toEqual([metadataModel]); + expect(request).toHaveBeenCalledOnce(); + }); + it("surfaces a first-load models.list failure instead of publishing an empty catalog", async () => { const request = vi.fn(async (method: string) => { if (method === "models.list") { @@ -1668,7 +1689,7 @@ describe("refreshChatMetadata", () => { }); const state = createMetadataState(request, { chatModelCatalog: [], - hello: { features: { methods: [] } }, + hello: { features: { methods: ["models.list"] } }, sessionKey: "agent:main:main", }); @@ -1715,7 +1736,7 @@ describe("refreshChatMetadata", () => { const state = createMetadataState(request, { agentsList: { defaultId: "main" } as ChatPageHost["agentsList"], chatModelCatalog: [{ id: "stale-model", name: "Stale Model", provider: "openai" }], - hello: { features: { methods: [] } }, + hello: { features: { methods: ["models.list"] } }, }); await refreshChatMetadata(state);