diff --git a/docs/concepts/models.md b/docs/concepts/models.md index cbcad3bc27af..0113d9a6a543 100644 --- a/docs/concepts/models.md +++ b/docs/concepts/models.md @@ -80,7 +80,7 @@ Other selection rules: - Changing `agents.defaults.model.primary` does not rewrite existing session pins. If status reports `This session is pinned to X; config primary Y will apply to new/unpinned sessions.`, run `/model default` to clear the pin. - CLI default-model and allowlist pickers respect `models.mode: "replace"` by listing only `models.providers.*.models` instead of the full built-in catalog. -- The Control UI model picker asks the Gateway for its configured model view. An explicit `modelPolicy.allow` filters it, including trailing prefix wildcard entries; otherwise it shows configured models plus providers with usable auth. Default and configured picker views hide catalog rows marked `deprecated` or `disabled` unless that exact model is configured as a primary, fallback, utility/tool model, alias/settings key, or exact policy entry. Hidden rows remain selectable by exact `provider/model` ref. The full built-in catalog, including hidden rows, is reserved for explicit browse views (`models.list` with `view: "all"`, or `openclaw models list --all`). +- The Control UI starts from the Gateway's prepared configured model view, so opening chat does not start provider discovery. Opening or refreshing a model picker may discover models required by a trailing `provider/*` policy entry. Default and configured picker views hide catalog rows marked `deprecated` or `disabled` unless that exact model is configured as a primary, fallback, utility/tool model, alias/settings key, or exact policy entry. Hidden rows remain selectable by exact `provider/model` ref. The full built-in catalog, including hidden rows, is reserved for explicit browse views (`models.list` with `view: "all"`, or `openclaw models list --all`). - Provider inventory UIs use `models.list` with `view: "provider-config"` to show source-authored `models.providers.*.models` rows without applying picker allowlists. Full mechanics: [Model failover](/concepts/model-failover). diff --git a/docs/gateway/protocol.md b/docs/gateway/protocol.md index 7433ee1827a2..c57e997f7f92 100644 --- a/docs/gateway/protocol.md +++ b/docs/gateway/protocol.md @@ -1029,6 +1029,15 @@ context. - `"all"`: full gateway catalog, bypassing `agents.defaults.modelPolicy.allow`. Use for diagnostics/discovery UIs, not normal model pickers. +Two optional controls separate automatic reads from operator-requested discovery: + +- `preparedOnly: true` reuses the current prepared catalog or a completed catalog for that + runtime generation without starting provider discovery. Control UI startup and polling use + this mode. +- `refresh: true` replaces a completed full catalog when the selected view requires discovery. + Concurrent refreshes share one build; a failed refresh leaves the previous completed catalog + available and returns the failure to the caller. + ## Exec approvals - When an exec request needs approval, the gateway broadcasts diff --git a/packages/gateway-protocol/src/index.test.ts b/packages/gateway-protocol/src/index.test.ts index 4c1ab617d706..f87b9e590650 100644 --- a/packages/gateway-protocol/src/index.test.ts +++ b/packages/gateway-protocol/src/index.test.ts @@ -1005,6 +1005,8 @@ describe("validateModelsListParams", () => { { view: "default" }, { view: "configured" }, { view: "all" }, + { view: "configured", preparedOnly: true }, + { view: "all", refresh: true }, ]); }); diff --git a/packages/gateway-protocol/src/schema/agents-models-skills.test.ts b/packages/gateway-protocol/src/schema/agents-models-skills.test.ts index 31e26286640f..69f87c6017ff 100644 --- a/packages/gateway-protocol/src/schema/agents-models-skills.test.ts +++ b/packages/gateway-protocol/src/schema/agents-models-skills.test.ts @@ -206,6 +206,13 @@ describe("ModelsListParamsSchema", () => { agentId: "research", includeProviderCapabilities: true, }, + { + preparedOnly: true, + }, + { + refresh: true, + view: "all", + }, ); expectRejected(ModelsListParamsSchema, { view: "provider-route" }, { agentId: "" }); }); diff --git a/packages/gateway-protocol/src/schema/agents-models-skills.ts b/packages/gateway-protocol/src/schema/agents-models-skills.ts index 99d3a86af5ff..b185b3e8d5d2 100644 --- a/packages/gateway-protocol/src/schema/agents-models-skills.ts +++ b/packages/gateway-protocol/src/schema/agents-models-skills.ts @@ -230,6 +230,10 @@ export const AgentsFilesSetResultSchema = closedObject({ export const ModelsListParamsSchema = closedObject({ agentId: Type.Optional(NonEmptyString), includeProviderCapabilities: Type.Optional(Type.Boolean()), + /** Reuse prepared/cached facts without starting provider discovery. */ + preparedOnly: Type.Optional(Type.Boolean()), + /** Force replacement of a completed full-catalog generation. */ + refresh: Type.Optional(Type.Boolean()), view: Type.Optional( Type.Union([ Type.Literal("default"), diff --git a/src/agents/agent-auth-discovery.ts b/src/agents/agent-auth-discovery.ts index 8fea556c4878..ba1835c3ce7d 100644 --- a/src/agents/agent-auth-discovery.ts +++ b/src/agents/agent-auth-discovery.ts @@ -22,6 +22,7 @@ export type DiscoverAuthStorageOptions = { ambientCredentials?: Readonly; externalCli?: ExternalCliAuthDiscovery; inheritedAuthDir?: string; + preparedStore?: AuthProfileStore; readOnly?: boolean; skipExternalAuthProfiles?: boolean; skipCredentials?: boolean; @@ -95,8 +96,9 @@ export function resolveAgentDiscoveryAuthFacts( ...(options?.externalCli ? { externalCli: options.externalCli } : {}), ...(options?.inheritedAuthDir ? { inheritedAuthDir: options.inheritedAuthDir } : {}), }; - const store = - options?.skipExternalAuthProfiles === true + const store = options?.preparedStore + ? options.preparedStore + : options?.skipExternalAuthProfiles === true ? ensureAuthProfileStoreWithoutExternalProfiles(agentDir, { allowKeychainPrompt: false, ...(options?.inheritedAuthDir ? { inheritedAuthDir: options.inheritedAuthDir } : {}), diff --git a/src/agents/auth-health.test.ts b/src/agents/auth-health.test.ts index 6d9fe4dbe523..a61b8dd7a2bf 100644 --- a/src/agents/auth-health.test.ts +++ b/src/agents/auth-health.test.ts @@ -6,11 +6,15 @@ import { MAX_DATE_TIMESTAMP_MS } from "@openclaw/normalization-core/number-coercion"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OAuthCredential } from "./auth-profiles/types.js"; +import type { ProviderAuthAliasLookupParams } from "./provider-auth-aliases.js"; -const { readCodexCliCredentialsCachedMock } = vi.hoisted(() => ({ +const { readCodexCliCredentialsCachedMock, resolveProviderIdForAuthMock } = vi.hoisted(() => ({ readCodexCliCredentialsCachedMock: vi.fn< (options?: { allowKeychainPrompt?: boolean }) => OAuthCredential | null >(() => null), + resolveProviderIdForAuthMock: vi.fn<(provider: string, params?: unknown) => string>( + (provider: string) => (provider === "codex-cli" ? "openai" : provider), + ), })); vi.mock("./cli-credentials.js", () => ({ @@ -20,7 +24,7 @@ vi.mock("./cli-credentials.js", () => ({ resetCliCredentialCachesForTest: () => undefined, })); vi.mock("./provider-auth-aliases.js", () => ({ - resolveProviderIdForAuth: (provider: string) => (provider === "codex-cli" ? "openai" : provider), + resolveProviderIdForAuth: resolveProviderIdForAuthMock, })); import { @@ -72,6 +76,10 @@ describe("buildAuthHealthSummary", () => { beforeEach(() => { readCodexCliCredentialsCachedMock.mockReset(); readCodexCliCredentialsCachedMock.mockReturnValue(null); + resolveProviderIdForAuthMock.mockReset(); + resolveProviderIdForAuthMock.mockImplementation((provider: string) => + provider === "codex-cli" ? "openai" : provider, + ); }); it("classifies OAuth and API key profiles", () => { @@ -580,6 +588,50 @@ describe("buildAuthHealthSummary", () => { }, ]); }); + + it("uses caller-owned plugin metadata when resolving explicit auth order", () => { + vi.spyOn(Date, "now").mockReturnValue(now); + resolveProviderIdForAuthMock.mockImplementation((provider: string, params?: unknown) => { + const metadata = (params as { metadataSnapshot?: { plugins?: unknown[] } } | undefined) + ?.metadataSnapshot; + return provider === "fixture-alias" && metadata?.plugins?.length + ? "fixture-provider" + : provider; + }); + const metadataSnapshot = { + plugins: [ + { + id: "fixture-auth-alias", + origin: "bundled" as const, + providerAuthAliases: { "fixture-alias": "fixture-provider" }, + }, + ], + } as unknown as NonNullable; + const summary = buildAuthHealthSummary({ + cfg: { auth: { order: { "fixture-provider": [] } } }, + store: { + version: 1, + profiles: { + "fixture-alias:token": { + type: "token", + provider: "fixture-alias", + token: "fake-token", + }, + }, + }, + authAliasLookupParams: { + metadataSnapshot, + }, + }); + + expect(summary.providers).toMatchObject([ + { provider: "fixture-alias", status: "missing", effectiveProfiles: [] }, + ]); + expect(resolveProviderIdForAuthMock).toHaveBeenCalledWith( + "fixture-alias", + expect.objectContaining({ metadataSnapshot }), + ); + }); }); describe("formatRemainingShort", () => { diff --git a/src/agents/auth-health.ts b/src/agents/auth-health.ts index 5648b09cd595..68df61a299d9 100644 --- a/src/agents/auth-health.ts +++ b/src/agents/auth-health.ts @@ -20,7 +20,10 @@ import { resolveAuthProfileDisplayLabel } from "./auth-profiles/display.js"; import { resolveEffectiveOAuthCredential } from "./auth-profiles/effective-oauth.js"; import { resolveAuthProfileOrder } from "./auth-profiles/order.js"; import type { AuthProfileCredential, AuthProfileStore } from "./auth-profiles/types.js"; -import { resolveProviderIdForAuth } from "./provider-auth-aliases.js"; +import { + type ProviderAuthAliasLookupParams, + resolveProviderIdForAuth, +} from "./provider-auth-aliases.js"; type AuthProfileSource = "store"; @@ -279,6 +282,8 @@ export function buildAuthHealthSummary(params: { providers?: string[]; runtimeCredentialsByProvider?: ReadonlyMap; allowKeychainPrompt?: boolean; + /** Exact prepared metadata for request paths that must not rediscover plugin aliases. */ + authAliasLookupParams?: ProviderAuthAliasLookupParams; }): AuthHealthSummary { const now = Date.now(); const warnAfterMs = params.warnAfterMs ?? DEFAULT_OAUTH_WARN_MS; @@ -338,7 +343,10 @@ export function buildAuthHealthSummary(params: { } const resolveExplicitAuthOrder = (provider: string): string[] | undefined => { - const authProvider = resolveProviderIdForAuth(provider, { config: params.cfg }); + const authProvider = resolveProviderIdForAuth(provider, { + config: params.cfg, + ...params.authAliasLookupParams, + }); return ( findNormalizedProviderValue(params.store.order, authProvider) ?? findNormalizedProviderValue(params.store.order, provider) ?? @@ -357,6 +365,7 @@ export function buildAuthHealthSummary(params: { cfg: params.cfg, store: params.store, provider: provider.provider, + authAliasLookupParams: params.authAliasLookupParams, }); const orderedProfiles = ordered .map((profileId) => provider.profiles.find((profile) => profile.profileId === profileId)) diff --git a/src/agents/auth-profiles/external-auth.ts b/src/agents/auth-profiles/external-auth.ts index c8efbb42dbac..c1eee78dc115 100644 --- a/src/agents/auth-profiles/external-auth.ts +++ b/src/agents/auth-profiles/external-auth.ts @@ -15,6 +15,11 @@ import { overlayRuntimeExternalOAuthProfiles, type RuntimeExternalOAuthProfile, } from "./oauth-shared.js"; +import { + getRuntimeExternalCliProfileIds, + removeRuntimeExternalProfileReferences, + setRuntimeExternalCliProfileIds, +} from "./runtime-external-profile-references.js"; import type { AuthProfileStore } from "./types.js"; type ExternalAuthProfileMap = Map; @@ -81,55 +86,38 @@ function isExternalAuthProfileAllowed( }); } -function resolveExternalAuthProfileMap(params: { +function resolveExternalAuthProfiles(params: { store: AuthProfileStore; agentDir?: string; - workspaceDir?: string; env?: NodeJS.ProcessEnv; externalCli?: ExternalCliOverlayOptions; -}): ExternalAuthProfileMap { +}): { + profiles: ExternalAuthProfileMap; + pluginProfileIds: ReadonlySet; + runtimeExternalCliProfileIds: ReadonlySet; +} { const env = params.env ?? process.env; const resolveProfiles = resolveExternalAuthProfilesForRuntime ?? resolveExternalAuthProfilesWithPlugins; const profiles = resolveProfiles({ env, config: params.externalCli?.config, - workspaceDir: params.workspaceDir, context: { config: params.externalCli?.config, agentDir: params.agentDir, - workspaceDir: params.workspaceDir, + workspaceDir: undefined, env, store: params.store, }, }); - - const resolved: ExternalAuthProfileMap = new Map(); + const resolved = resolveExternalCliAuthProfileMap(params); + const runtimeExternalCliProfileIds = new Set( + [...resolved.values()] + .filter((profile) => profile.persistence !== "persisted") + .map((profile) => profile.profileId), + ); + const pluginProfileIds = new Set(); const explicitProfileIds = resolveExplicitProfileIds(params.externalCli?.externalCliProfileIds); - const cliProfiles = - externalCliSync.resolveExternalCliAuthProfiles?.(params.store, { - allowKeychainPrompt: params.externalCli?.allowKeychainPrompt, - providerIds: params.externalCli?.externalCliProviderIds, - profileIds: explicitProfileIds, - }) ?? []; - for (const profile of cliProfiles) { - if ( - !isExternalAuthProfileAllowed( - profile, - params.store, - params.externalCli?.config, - explicitProfileIds, - env, - ) - ) { - continue; - } - resolved.set(profile.profileId, { - profileId: profile.profileId, - credential: profile.credential, - persistence: profile.persistence ?? "runtime-only", - }); - } for (const rawProfile of profiles) { const profile = normalizeExternalAuthProfile(rawProfile); if (!profile) { @@ -147,26 +135,68 @@ function resolveExternalAuthProfileMap(params: { continue; } resolved.set(profile.profileId, profile); + pluginProfileIds.add(profile.profileId); + runtimeExternalCliProfileIds.delete(profile.profileId); } - return resolved; + return { profiles: resolved, pluginProfileIds, runtimeExternalCliProfileIds }; +} + +function resolveAllowedExternalCliAuthProfiles(params: { + store: AuthProfileStore; + env?: NodeJS.ProcessEnv; + externalCli?: ExternalCliOverlayOptions; +}): ProviderExternalAuthProfile[] { + const env = params.env ?? process.env; + const explicitProfileIds = resolveExplicitProfileIds(params.externalCli?.externalCliProfileIds); + const cliProfiles = + externalCliSync.resolveExternalCliAuthProfiles?.(params.store, { + allowKeychainPrompt: params.externalCli?.allowKeychainPrompt, + providerIds: params.externalCli?.externalCliProviderIds, + profileIds: explicitProfileIds, + }) ?? []; + return cliProfiles.flatMap((profile) => + isExternalAuthProfileAllowed( + profile, + params.store, + params.externalCli?.config, + explicitProfileIds, + env, + ) + ? [ + { + profileId: profile.profileId, + credential: profile.credential, + persistence: profile.persistence ?? "runtime-only", + }, + ] + : [], + ); +} + +function resolveExternalCliAuthProfileMap(params: { + store: AuthProfileStore; + env?: NodeJS.ProcessEnv; + externalCli?: ExternalCliOverlayOptions; +}): ExternalAuthProfileMap { + return new Map( + resolveAllowedExternalCliAuthProfiles(params).map((profile) => [profile.profileId, profile]), + ); } /** List runtime-only and persisted external auth profiles for this store. */ export function listRuntimeExternalAuthProfiles(params: { store: AuthProfileStore; agentDir?: string; - workspaceDir?: string; env?: NodeJS.ProcessEnv; externalCli?: ExternalCliOverlayOptions; }): RuntimeExternalOAuthProfile[] { return Array.from( - resolveExternalAuthProfileMap({ + resolveExternalAuthProfiles({ store: params.store, agentDir: params.agentDir, - workspaceDir: params.workspaceDir, env: params.env, externalCli: params.externalCli, - }).values(), + }).profiles.values(), ); } @@ -193,22 +223,73 @@ function hasScopedExternalCliOverlay(params?: ExternalCliOverlayOptions): boolea /** Overlay external auth profiles onto a cloned auth store for runtime use. */ export function overlayExternalAuthProfiles( store: AuthProfileStore, - params?: { - agentDir?: string; - workspaceDir?: string; - env?: NodeJS.ProcessEnv; - } & ExternalCliOverlayOptions, + params?: { agentDir?: string; env?: NodeJS.ProcessEnv } & ExternalCliOverlayOptions, ): AuthProfileStore { - const profiles = listRuntimeExternalAuthProfiles({ - store, + const scoped = hasScopedExternalCliOverlay(params); + const refreshedProfileIds = new Set( + getRuntimeExternalCliProfileIds(store).filter( + (profileId) => + scoped && + externalCliSync.isExternalCliAuthProfileInScope({ + store, + profileId, + providerIds: params?.externalCliProviderIds, + profileIds: params?.externalCliProfileIds, + }), + ), + ); + const base = removeRuntimeExternalProfileReferences({ store, profileIds: refreshedProfileIds }); + const resolved = resolveExternalAuthProfiles({ + store: base, agentDir: params?.agentDir, - workspaceDir: params?.workspaceDir, env: params?.env, externalCli: params, }); - return overlayRuntimeExternalOAuthProfiles(store, profiles, { - runtimeExternalProfileIdsAuthoritative: !hasScopedExternalCliOverlay(params), + const next = overlayRuntimeExternalOAuthProfiles(base, resolved.profiles.values(), { + runtimeExternalProfileIdsAuthoritative: !scoped, }); + const retainedCliProfileIds = getRuntimeExternalCliProfileIds(base).filter( + (profileId) => !resolved.pluginProfileIds.has(profileId), + ); + setRuntimeExternalCliProfileIds(next, [ + ...retainedCliProfileIds, + ...resolved.runtimeExternalCliProfileIds, + ]); + return next; +} + +/** Refresh external CLI credentials without reevaluating lifecycle-owned provider hooks. */ +export function overlayExternalCliAuthProfiles( + store: AuthProfileStore, + params?: { env?: NodeJS.ProcessEnv } & ExternalCliOverlayOptions, +): AuthProfileStore { + const scoped = hasScopedExternalCliOverlay(params); + const refreshedProfileIds = new Set( + getRuntimeExternalCliProfileIds(store).filter( + (profileId) => + scoped && + externalCliSync.isExternalCliAuthProfileInScope({ + store, + profileId, + providerIds: params?.externalCliProviderIds, + profileIds: params?.externalCliProfileIds, + }), + ), + ); + const base = removeRuntimeExternalProfileReferences({ store, profileIds: refreshedProfileIds }); + const profiles = resolveAllowedExternalCliAuthProfiles({ + store: base, + env: params?.env, + externalCli: params, + }); + const next = overlayRuntimeExternalOAuthProfiles(base, profiles); + setRuntimeExternalCliProfileIds(next, [ + ...getRuntimeExternalCliProfileIds(base), + ...profiles + .filter((profile) => profile.persistence !== "persisted") + .map((profile) => profile.profileId), + ]); + return next; } /** Persist safe external CLI OAuth profiles that own their local profile slot. */ @@ -219,19 +300,11 @@ export function syncPersistedExternalCliAuthProfiles( if (!hasPersistableExternalCliSyncCandidate(store, params)) { return store; } - const env = params?.env ?? process.env; - const explicitProfileIds = resolveExplicitProfileIds(params?.externalCliProfileIds); - const cliProfiles = - externalCliSync.resolveExternalCliAuthProfiles?.(store, { - allowKeychainPrompt: params?.allowKeychainPrompt, - providerIds: params?.externalCliProviderIds, - profileIds: explicitProfileIds, - }) ?? []; - const persistedProfiles = cliProfiles.filter( - (profile) => - profile.persistence === "persisted" && - isExternalAuthProfileAllowed(profile, store, params?.config, explicitProfileIds, env), - ); + const persistedProfiles = resolveAllowedExternalCliAuthProfiles({ + store, + env: params?.env, + externalCli: params, + }).filter((profile) => profile.persistence === "persisted"); if (persistedProfiles.length === 0) { return store; } diff --git a/src/agents/auth-profiles/external-cli-sync.ts b/src/agents/auth-profiles/external-cli-sync.ts index 48db41611195..26ac9c06e7e9 100644 --- a/src/agents/auth-profiles/external-cli-sync.ts +++ b/src/agents/auth-profiles/external-cli-sync.ts @@ -266,6 +266,30 @@ function isExternalCliProviderInScope(params: { }); } +/** True when a previously resolved built-in CLI profile belongs to this refresh scope. */ +export function isExternalCliAuthProfileInScope(params: { + store: AuthProfileStore; + profileId: string; + providerIds?: Iterable; + profileIds?: Iterable; +}): boolean { + const credential = params.store.profiles[params.profileId]; + const providerConfig = resolveExternalCliSyncProvider({ + profileId: params.profileId, + ...(credential?.type === "oauth" ? { credential } : {}), + }); + return providerConfig + ? isExternalCliProviderInScope({ + providerConfig, + store: params.store, + options: { + ...(params.providerIds ? { providerIds: params.providerIds } : {}), + ...(params.profileIds ? { profileIds: params.profileIds } : {}), + }, + }) + : false; +} + function listScopedExternalCliProfileIds(params: { providerConfig: ExternalCliSyncProvider; store: AuthProfileStore; diff --git a/src/agents/auth-profiles/external-oauth.test.ts b/src/agents/auth-profiles/external-oauth.test.ts index ec3c79385635..510963cf8d83 100644 --- a/src/agents/auth-profiles/external-oauth.test.ts +++ b/src/agents/auth-profiles/external-oauth.test.ts @@ -8,16 +8,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ProviderExternalAuthProfile } from "../../plugins/types.js"; import { resolveAgentCredentialMapFromStore } from "../agent-auth-credentials.js"; import { addEnvBackedAgentCredentials } from "../agent-auth-discovery-core.js"; -import { overlayExternalAuthProfiles } from "./external-auth.js"; +import { overlayExternalAuthProfiles, overlayExternalCliAuthProfiles } from "./external-auth.js"; import { testing } from "./external-auth.test-support.js"; import { readExternalCliBootstrapCredential } from "./external-cli-sync.js"; +import { getRuntimeExternalCliProfileIds } from "./runtime-external-profile-references.js"; import { clearRuntimeAuthProfileStoreSnapshots, registerRuntimeAuthProfileStoreMutationListener, replaceRuntimeAuthProfileStoreSnapshots, } from "./runtime-snapshots.js"; import { ensureAuthProfileStore, getRuntimeAuthProfileStoreSnapshot } from "./store.js"; -import type { AuthProfileStore, OAuthCredential } from "./types.js"; +import type { AuthProfileStore, OAuthCredential, RuntimeAuthProfileStore } from "./types.js"; const resolveExternalAuthProfilesWithPluginsMock = vi.fn< (params: unknown) => ProviderExternalAuthProfile[] @@ -113,6 +114,109 @@ describe("auth external oauth helpers", () => { expect(readCodexCliCredentialsCachedMock).toHaveBeenCalledTimes(1); }); + it("refreshes and removes a prepared built-in CLI profile authoritatively", () => { + readCodexCliCredentialsCachedMock.mockReturnValueOnce( + createCredential({ access: "startup-access", refresh: "startup-refresh" }), + ); + const startup = overlayExternalAuthProfiles( + { + ...createStore(), + order: { openai: ["openai:default"] }, + lastGood: { openai: "openai:default" }, + usageStats: { "openai:default": { lastUsed: 1 } }, + }, + { + externalCliProviderIds: ["openai"], + }, + ); + expect(getRuntimeExternalCliProfileIds(startup)).toEqual(["openai:default"]); + + const retained = overlayExternalAuthProfiles(startup); + expect(retained.profiles["openai:default"]).toMatchObject({ + access: "startup-access", + refresh: "startup-refresh", + }); + expect(getRuntimeExternalCliProfileIds(retained)).toEqual(["openai:default"]); + + readCodexCliCredentialsCachedMock.mockReturnValueOnce( + createCredential({ access: "rotated-access", refresh: "rotated-refresh" }), + ); + const rotated = overlayExternalCliAuthProfiles(retained, { + externalCliProfileIds: ["openai:default"], + }); + expect(rotated.profiles["openai:default"]).toMatchObject({ + access: "rotated-access", + refresh: "rotated-refresh", + }); + expect(getRuntimeExternalCliProfileIds(rotated)).toEqual(["openai:default"]); + + readCodexCliCredentialsCachedMock.mockReturnValueOnce(null); + const loggedOut = overlayExternalCliAuthProfiles(rotated, { + externalCliProviderIds: ["openai"], + }); + expect(loggedOut.profiles["openai:default"]).toBeUndefined(); + expect(loggedOut.order).toBeUndefined(); + expect(loggedOut.lastGood).toBeUndefined(); + expect(loggedOut.usageStats).toBeUndefined(); + expect(getRuntimeExternalCliProfileIds(loggedOut)).toEqual([]); + }); + + it("preserves a plugin winner that collides with a built-in CLI profile id", () => { + readCodexCliCredentialsCachedMock.mockReturnValue( + createCredential({ access: "cli-access", refresh: "cli-refresh" }), + ); + resolveExternalAuthProfilesWithPluginsMock.mockReturnValue([ + { + profileId: "openai:default", + credential: createCredential({ access: "plugin-access", refresh: "plugin-refresh" }), + }, + ]); + const prepared = overlayExternalAuthProfiles(createStore(), { + externalCliProviderIds: ["openai"], + }); + expect(prepared.profiles["openai:default"]).toMatchObject({ + access: "plugin-access", + refresh: "plugin-refresh", + }); + expect(prepared.runtimeExternalProfileIds).toEqual(["openai:default"]); + expect(getRuntimeExternalCliProfileIds(prepared)).toEqual([]); + + resolveExternalAuthProfilesWithPluginsMock.mockClear(); + const refreshed = overlayExternalCliAuthProfiles(prepared, { + externalCliProviderIds: ["openai"], + }); + expect(refreshed.profiles["openai:default"]).toMatchObject({ + access: "plugin-access", + refresh: "plugin-refresh", + }); + expect(resolveExternalAuthProfilesWithPluginsMock).not.toHaveBeenCalled(); + }); + + it("replaces CLI provenance only inside the requested refresh scope", () => { + const store: RuntimeAuthProfileStore = { + ...createStore({ + "openai:default": createCredential(), + "claude-cli:default": createCredential({ + provider: "claude-cli", + access: "claude-access", + refresh: "claude-refresh", + }), + }), + runtimeExternalProfileIds: ["claude-cli:default", "openai:default"], + runtimeExternalCliProfileIds: ["claude-cli:default", "openai:default"], + }; + + const refreshed = overlayExternalCliAuthProfiles(store, { + externalCliProfileIds: ["openai:default"], + }); + + expect(refreshed.profiles["openai:default"]).toBeUndefined(); + expect(refreshed.profiles["claude-cli:default"]).toMatchObject({ + access: "claude-access", + }); + expect(getRuntimeExternalCliProfileIds(refreshed)).toEqual(["claude-cli:default"]); + }); + it("publishes a usable scoped CLI bootstrap into the runtime auth owner", () => { const agentDir = "/tmp/openclaw-external-oauth-publication"; readCodexCliCredentialsCachedMock.mockReturnValue( diff --git a/src/agents/auth-profiles/order.ts b/src/agents/auth-profiles/order.ts index b81824c2ab60..199a1b6d3b29 100644 --- a/src/agents/auth-profiles/order.ts +++ b/src/agents/auth-profiles/order.ts @@ -129,6 +129,7 @@ function isConfiguredProfileCompatibleWithAuthProvider(params: { function listProfilesCompatibleWithAuthProvider(params: { cfg?: OpenClawConfig; + authAliasLookupParams?: ProviderAuthAliasLookupParams; store: AuthProfileStore; provider: string; providerAuthKey: string; @@ -140,6 +141,7 @@ function listProfilesCompatibleWithAuthProvider(params: { .filter(([, credential]) => isCredentialProviderCompatibleWithAuthProvider({ cfg: params.cfg, + authAliasLookupParams: params.authAliasLookupParams, providerAuthKey: params.providerAuthKey, credential, }), @@ -263,6 +265,8 @@ type ResolveAuthProfileOrderParams = { cfg?: OpenClawConfig; store: AuthProfileStore; provider: string; + /** Exact prepared metadata for request paths that must not rediscover plugin aliases. */ + authAliasLookupParams?: ProviderAuthAliasLookupParams; preferredProfile?: string; /** Model that will consume the profile, for model-scoped cooldowns. */ forModel?: string; @@ -282,7 +286,10 @@ export function resolveAuthProfileOrderWithMetadata( ): AuthProfileOrderResolution { const { cfg, store, provider, preferredProfile, forModel } = params; const providerKey = normalizeProviderId(provider); - const providerAuthKey = resolveProviderIdForAuth(provider, { config: cfg }); + const providerAuthKey = resolveProviderIdForAuth(provider, { + config: cfg, + ...params.authAliasLookupParams, + }); const now = Date.now(); // Clear any cooldowns that have expired since the last check so profiles @@ -316,6 +323,7 @@ export function resolveAuthProfileOrderWithMetadata( .filter(([profileId, profile]) => isConfiguredProfileCompatibleWithAuthProvider({ cfg, + authAliasLookupParams: params.authAliasLookupParams, providerAuthKey, provider: profile.provider, mode: profile.mode, @@ -326,6 +334,7 @@ export function resolveAuthProfileOrderWithMetadata( : []; const storeProfiles = listProfilesCompatibleWithAuthProvider({ cfg, + authAliasLookupParams: params.authAliasLookupParams, store, provider, providerAuthKey, @@ -335,6 +344,7 @@ export function resolveAuthProfileOrderWithMetadata( ? storeProfiles.filter((profileId) => isNativeCredentialProviderCompatibleWithAuthProvider({ cfg, + authAliasLookupParams: params.authAliasLookupParams, providerAuthKey, credential: store.profiles[profileId], }), @@ -357,6 +367,7 @@ export function resolveAuthProfileOrderWithMetadata( const isValidProfile = (profileId: string): boolean => { const eligibility = resolveAuthProfileEligibility({ cfg, + authAliasLookupParams: params.authAliasLookupParams, store, provider, profileId, @@ -445,6 +456,7 @@ function resolveAuthOrder( function isNativeCredentialProviderCompatibleWithAuthProvider(params: { cfg?: OpenClawConfig; + authAliasLookupParams?: ProviderAuthAliasLookupParams; providerAuthKey: string; credential: AuthProfileCredential | undefined; }): boolean { @@ -452,8 +464,10 @@ function isNativeCredentialProviderCompatibleWithAuthProvider(params: { return false; } return ( - resolveProviderIdForAuth(params.credential.provider, { config: params.cfg }) === - params.providerAuthKey + resolveProviderIdForAuth(params.credential.provider, { + config: params.cfg, + ...params.authAliasLookupParams, + }) === params.providerAuthKey ); } diff --git a/src/agents/auth-profiles/persisted-boundary.test.ts b/src/agents/auth-profiles/persisted-boundary.test.ts index 1d0ab625c386..a8cb06fa5ee7 100644 --- a/src/agents/auth-profiles/persisted-boundary.test.ts +++ b/src/agents/auth-profiles/persisted-boundary.test.ts @@ -15,7 +15,8 @@ import { coercePersistedAuthProfileStore, mergeAuthProfileStores, } from "./persisted.js"; -import type { AuthProfileStore } from "./types.js"; +import { getRuntimeExternalCliProfileIds } from "./runtime-external-profile-references.js"; +import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js"; describe("persisted auth profile boundary", () => { it("normalizes malformed persisted credentials and state before runtime use", () => { @@ -428,6 +429,47 @@ describe("persisted auth profile boundary", () => { expect(merged.order?.anthropic).toEqual([profileId]); expect(merged.lastGood?.anthropic).toBe(profileId); }); + + it("carries built-in CLI provenance only with the winning external profile", () => { + const profileId = "openai:default"; + const base: RuntimeAuthProfileStore = { + version: AUTH_STORE_VERSION, + runtimeExternalProfileIds: [profileId], + runtimeExternalCliProfileIds: [profileId], + profiles: { + [profileId]: { + type: "oauth", + provider: "openai", + access: "cli-access", + refresh: "cli-refresh", + expires: 1, + }, + }, + }; + const inherited = mergeAuthProfileStores( + base, + { version: AUTH_STORE_VERSION, profiles: {} }, + { preserveBaseRuntimeExternalProfiles: true }, + ); + expect(getRuntimeExternalCliProfileIds(inherited)).toEqual([profileId]); + + const pluginOverride: RuntimeAuthProfileStore = { + version: AUTH_STORE_VERSION, + runtimeExternalProfileIds: [profileId], + profiles: { + [profileId]: { + type: "oauth", + provider: "openai", + access: "plugin-access", + refresh: "plugin-refresh", + expires: 2, + }, + }, + }; + const collided = mergeAuthProfileStores(base, pluginOverride); + expect(collided.profiles[profileId]).toMatchObject({ access: "plugin-access" }); + expect(getRuntimeExternalCliProfileIds(collided)).toEqual([]); + }); }); describe("applyLegacyAuthStore", () => { diff --git a/src/agents/auth-profiles/persisted.ts b/src/agents/auth-profiles/persisted.ts index 13a98ad9d8a1..c67445c4d054 100644 --- a/src/agents/auth-profiles/persisted.ts +++ b/src/agents/auth-profiles/persisted.ts @@ -19,6 +19,10 @@ import { normalizeAuthEmailToken, normalizeAuthIdentityToken, } from "./oauth-shared.js"; +import { + getRuntimeExternalCliProfileIds, + setRuntimeExternalCliProfileIds, +} from "./runtime-external-profile-references.js"; import { readPersistedAuthProfileStoreRaw } from "./sqlite.js"; import { coerceAuthProfileState, @@ -555,7 +559,7 @@ function replaceMergedProfileReferences(params: { } } - return { + const next = { ...store, profiles, ...(order && Object.keys(order).length > 0 ? { order } : { order: undefined }), @@ -564,6 +568,13 @@ function replaceMergedProfileReferences(params: { ? { usageStats } : { usageStats: undefined }), }; + setRuntimeExternalCliProfileIds( + next, + getRuntimeExternalCliProfileIds(store).map( + (profileId) => replacements.get(profileId) ?? profileId, + ), + ); + return next; } function reconcileMainStoreOAuthProfileDrift(params: { @@ -609,7 +620,8 @@ export function mergeAuthProfileStores( override.runtimeLocalProfileIds === undefined && override.runtimeInheritsMainState === undefined && override.runtimeExternalProfileIds === undefined && - override.runtimeExternalProfileIdsAuthoritative !== true + override.runtimeExternalProfileIdsAuthoritative !== true && + getRuntimeExternalCliProfileIds(override).length === 0 ) { return base; } @@ -705,7 +717,14 @@ export function mergeAuthProfileStores( : {}), } : {}; - return reconcileMainStoreOAuthProfileDrift({ + const runtimeExternalCliProfileIds = [ + ...getRuntimeExternalCliProfileIds(base).filter( + (profileId) => + !overrideProfileIds.has(profileId) && !removedRuntimeExternalProfileIds.has(profileId), + ), + ...getRuntimeExternalCliProfileIds(override), + ]; + const result = reconcileMainStoreOAuthProfileDrift({ base, override, merged: { @@ -720,6 +739,8 @@ export function mergeAuthProfileStores( ...runtimeExternalProfileMetadata, }, }) as RuntimeAuthProfileStore; + setRuntimeExternalCliProfileIds(result, runtimeExternalCliProfileIds); + return result; } /** Builds the persisted secrets store, stripping resolved literals when refs exist. */ diff --git a/src/agents/auth-profiles/profiles.test.ts b/src/agents/auth-profiles/profiles.test.ts index 98a145c4ea3f..5ff14d48a86d 100644 --- a/src/agents/auth-profiles/profiles.test.ts +++ b/src/agents/auth-profiles/profiles.test.ts @@ -599,6 +599,33 @@ describe("promoteAuthProfileInOrder", () => { }); }); + it("does not persist built-in CLI ownership metadata", async () => { + await withAuthProfileTestState("openclaw-auth-cli-provenance-", async ({ agentDir }) => { + const profileId = "openai:default"; + const runtimeStore: RuntimeAuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: { + [profileId]: { + type: "oauth", + provider: "openai", + access: "external-access", + refresh: "external-refresh", + expires: Date.now() + 60_000, + }, + }, + runtimeExternalProfileIds: [profileId], + runtimeExternalCliProfileIds: [profileId], + }; + replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store: runtimeStore }]); + + saveAuthProfileStore(runtimeStore, agentDir); + + const persisted = loadPersistedAuthProfileStore(agentDir); + expect(persisted).not.toHaveProperty("runtimeExternalCliProfileIds"); + expect(persisted?.profiles[profileId]).toBeUndefined(); + }); + }); + it.each(["before save", "before publication"] as const)( "preserves a runtime-only OAuth mutation %s", async (mutationTiming) => { diff --git a/src/agents/auth-profiles/runtime-external-profile-references.ts b/src/agents/auth-profiles/runtime-external-profile-references.ts new file mode 100644 index 000000000000..96b1210fca44 --- /dev/null +++ b/src/agents/auth-profiles/runtime-external-profile-references.ts @@ -0,0 +1,160 @@ +import { isDeepStrictEqual } from "node:util"; +import { cloneAuthProfileStore } from "./clone.js"; +import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js"; + +type RuntimeExternalCliStore = AuthProfileStore & + Pick; + +export function getRuntimeExternalCliProfileIds(store: AuthProfileStore): readonly string[] { + return (store as RuntimeExternalCliStore).runtimeExternalCliProfileIds ?? []; +} + +export function setRuntimeExternalCliProfileIds( + store: AuthProfileStore, + profileIds: Iterable, +): void { + const ids = [...new Set(profileIds)].filter((profileId) => store.profiles[profileId]).toSorted(); + (store as RuntimeExternalCliStore).runtimeExternalCliProfileIds = + ids.length > 0 ? ids : undefined; +} + +export function removeRuntimeExternalProfileReferences(params: { + store: AuthProfileStore; + profileIds: ReadonlySet; +}): AuthProfileStore { + if (params.profileIds.size === 0) { + return params.store; + } + const next = cloneAuthProfileStore(params.store); + for (const profileId of params.profileIds) { + delete next.profiles[profileId]; + if (next.usageStats) { + delete next.usageStats[profileId]; + } + } + if (next.order) { + const order = Object.fromEntries( + Object.entries(next.order) + .map( + ([provider, profileIds]) => + [ + provider, + profileIds.filter((profileId) => !params.profileIds.has(profileId)), + ] as const, + ) + .filter(([, profileIds]) => profileIds.length > 0), + ); + next.order = Object.keys(order).length > 0 ? order : undefined; + } + if (next.lastGood) { + const lastGood = Object.fromEntries( + Object.entries(next.lastGood).filter(([, profileId]) => !params.profileIds.has(profileId)), + ); + next.lastGood = Object.keys(lastGood).length > 0 ? lastGood : undefined; + } + if (next.usageStats && Object.keys(next.usageStats).length === 0) { + next.usageStats = undefined; + } + next.runtimePersistedProfileIds = next.runtimePersistedProfileIds?.filter( + (profileId) => !params.profileIds.has(profileId), + ); + if (next.runtimePersistedProfileIds?.length === 0) { + next.runtimePersistedProfileIds = undefined; + } + next.runtimeExternalProfileIds = next.runtimeExternalProfileIds?.filter( + (profileId) => !params.profileIds.has(profileId), + ); + if ( + next.runtimeExternalProfileIds?.length === 0 && + next.runtimeExternalProfileIdsAuthoritative !== true + ) { + next.runtimeExternalProfileIds = undefined; + } + setRuntimeExternalCliProfileIds( + next, + getRuntimeExternalCliProfileIds(next).filter((profileId) => !params.profileIds.has(profileId)), + ); + return next; +} + +/** Carries lifecycle-owned external profiles across a durable-store refresh. */ +export function mergeRuntimeExternalProfileReferences(params: { + next: AuthProfileStore; + existing: AuthProfileStore; +}): AuthProfileStore { + const runtimeExternalProfileIds = new Set(params.existing.runtimeExternalProfileIds ?? []); + if (params.next.runtimeExternalProfileIdsAuthoritative === true) { + return params.next; + } + if (runtimeExternalProfileIds.size === 0) { + return params.next; + } + const merged = cloneAuthProfileStore(params.next); + const mergedRuntimeExternalProfileIds = new Set(merged.runtimeExternalProfileIds ?? []); + const mergedRuntimeExternalCliProfileIds = new Set(getRuntimeExternalCliProfileIds(merged)); + const existingRuntimeExternalCliProfileIds = new Set( + getRuntimeExternalCliProfileIds(params.existing), + ); + const backfilledRuntimeExternalProfileIds = new Set(); + for (const profileId of runtimeExternalProfileIds) { + const existingCredential = params.existing.profiles[profileId]; + const nextCredential = merged.profiles[profileId]; + if (nextCredential) { + if ( + mergedRuntimeExternalProfileIds.has(profileId) || + (existingCredential && isDeepStrictEqual(nextCredential, existingCredential)) + ) { + mergedRuntimeExternalProfileIds.add(profileId); + if (existingRuntimeExternalCliProfileIds.has(profileId)) { + mergedRuntimeExternalCliProfileIds.add(profileId); + } + } + continue; + } + if (!existingCredential) { + continue; + } + merged.profiles[profileId] = existingCredential; + mergedRuntimeExternalProfileIds.add(profileId); + if (existingRuntimeExternalCliProfileIds.has(profileId)) { + mergedRuntimeExternalCliProfileIds.add(profileId); + } + backfilledRuntimeExternalProfileIds.add(profileId); + if (params.existing.usageStats?.[profileId]) { + merged.usageStats = { + ...merged.usageStats, + [profileId]: params.existing.usageStats[profileId], + }; + } + } + for (const [provider, profileIds] of Object.entries(params.existing.order ?? {})) { + const externalProfileIds = profileIds.filter((profileId) => + backfilledRuntimeExternalProfileIds.has(profileId), + ); + if (externalProfileIds.length === 0 || merged.order?.[provider]) { + continue; + } + merged.order = { + ...merged.order, + [provider]: externalProfileIds, + }; + } + for (const [provider, profileId] of Object.entries(params.existing.lastGood ?? {})) { + if (!backfilledRuntimeExternalProfileIds.has(profileId) || merged.lastGood?.[provider]) { + continue; + } + merged.lastGood = { + ...merged.lastGood, + [provider]: profileId, + }; + } + const profileIds = [...mergedRuntimeExternalProfileIds].toSorted(); + merged.runtimeExternalProfileIds = + profileIds.length > 0 || params.existing.runtimeExternalProfileIdsAuthoritative === true + ? profileIds + : undefined; + merged.runtimeExternalProfileIdsAuthoritative = + params.existing.runtimeExternalProfileIdsAuthoritative === true ? true : undefined; + setRuntimeExternalCliProfileIds(merged, mergedRuntimeExternalCliProfileIds); + return merged; +} diff --git a/src/agents/auth-profiles/runtime-snapshots.test.ts b/src/agents/auth-profiles/runtime-snapshots.test.ts index 03cc941b98fb..b1c0a0eddc55 100644 --- a/src/agents/auth-profiles/runtime-snapshots.test.ts +++ b/src/agents/auth-profiles/runtime-snapshots.test.ts @@ -25,7 +25,7 @@ import { setRuntimeAuthProfileStoreSnapshot, } from "./runtime-snapshots.js"; import { testing } from "./runtime-snapshots.test-support.js"; -import type { AuthProfileStore } from "./types.js"; +import type { AuthProfileStore, RuntimeAuthProfileStore } from "./types.js"; function createStore(access: string): AuthProfileStore { return { @@ -240,6 +240,36 @@ describe("runtime auth profile snapshots", () => { } }); + it("notifies when identical external credentials change from CLI to plugin ownership", () => { + const agentDir = "/tmp/openclaw-auth-runtime-external-owner"; + const store: RuntimeAuthProfileStore = { + ...createStore("same-credential"), + runtimeExternalProfileIds: ["openai:default"], + runtimeExternalCliProfileIds: ["openai:default"], + }; + setRuntimeAuthProfileStoreSnapshot(store, agentDir); + const listener = vi.fn(); + const unregister = registerRuntimeAuthProfileStoreMutationListener(listener); + try { + const pluginOwned: RuntimeAuthProfileStore = { + ...store, + runtimeExternalCliProfileIds: undefined, + }; + replaceRuntimeAuthProfileStoreSnapshots([ + { + agentDir, + store: pluginOwned, + }, + ]); + + expect(listener).toHaveBeenCalledOnce(); + expect(listener).toHaveBeenCalledWith({ affectsInheritedStores: true }); + } finally { + unregister(); + clearRuntimeAuthProfileStoreSnapshots(); + } + }); + it("notifies when an empty runtime snapshot starts or stops shadowing persisted auth", () => { const agentDir = "/tmp/openclaw-auth-runtime-empty-owner"; const listener = vi.fn(); diff --git a/src/agents/auth-profiles/runtime-snapshots.ts b/src/agents/auth-profiles/runtime-snapshots.ts index d81fa484323d..f8219fb13772 100644 --- a/src/agents/auth-profiles/runtime-snapshots.ts +++ b/src/agents/auth-profiles/runtime-snapshots.ts @@ -144,6 +144,7 @@ function ownerState( | "runtimePersistedProfileIds" | "runtimeExternalProfileIds" | "runtimeExternalProfileIdsAuthoritative" + | "runtimeExternalCliProfileIds" | "runtimeLocalProfileIds" | "runtimeInheritsMainState" > @@ -157,6 +158,7 @@ function ownerState( runtimePersistedProfileIds: store.runtimePersistedProfileIds, runtimeExternalProfileIds: store.runtimeExternalProfileIds, runtimeExternalProfileIdsAuthoritative: store.runtimeExternalProfileIdsAuthoritative, + runtimeExternalCliProfileIds: store.runtimeExternalCliProfileIds, runtimeLocalProfileIds: store.runtimeLocalProfileIds, runtimeInheritsMainState: store.runtimeInheritsMainState, }; diff --git a/src/agents/auth-profiles/store.ts b/src/agents/auth-profiles/store.ts index aa9a2cc68d63..40cf04078dfd 100644 --- a/src/agents/auth-profiles/store.ts +++ b/src/agents/auth-profiles/store.ts @@ -44,6 +44,11 @@ import { loadPersistedAuthProfileStore, mergeAuthProfileStores, } from "./persisted.js"; +import { + getRuntimeExternalCliProfileIds, + mergeRuntimeExternalProfileReferences, + setRuntimeExternalCliProfileIds, +} from "./runtime-external-profile-references.js"; import { clearRuntimeAuthProfileStoreSnapshotCore, clearRuntimeAuthProfileStoreSnapshots, @@ -545,6 +550,10 @@ function pruneAuthProfileStoreReferences( store.runtimeExternalProfileIds = store.runtimeExternalProfileIds ?.filter((profileId) => keptProfileIds.has(profileId)) .toSorted(); + setRuntimeExternalCliProfileIds( + store, + getRuntimeExternalCliProfileIds(store).filter((profileId) => keptProfileIds.has(profileId)), + ); if ( store.runtimeExternalProfileIds?.length === 0 && store.runtimeExternalProfileIdsAuthoritative !== true @@ -616,6 +625,7 @@ function buildLocalAuthProfileStoreForSave(params: { if (params.options?.filterExternalAuthProfiles !== false) { localStore.runtimeExternalProfileIds = undefined; localStore.runtimeExternalProfileIdsAuthoritative = undefined; + setRuntimeExternalCliProfileIds(localStore, []); } return localStore; } @@ -646,6 +656,7 @@ function stripRuntimeExternalProfileMetadata(store: AuthProfileStore): AuthProfi const stripped = { ...store }; delete stripped.runtimeExternalProfileIds; delete stripped.runtimeExternalProfileIdsAuthoritative; + setRuntimeExternalCliProfileIds(stripped, []); return stripped; } @@ -727,81 +738,12 @@ function setRuntimeExternalProfileMetadata(params: { params.store.runtimeExternalProfileIds = profileIds.length > 0 || params.authoritative ? profileIds : undefined; params.store.runtimeExternalProfileIdsAuthoritative = params.authoritative ? true : undefined; -} - -function mergeRuntimeExternalProfileReferences(params: { - next: AuthProfileStore; - existing: AuthProfileStore; -}): AuthProfileStore { - const runtimeExternalProfileIds = new Set(params.existing.runtimeExternalProfileIds ?? []); - if (params.next.runtimeExternalProfileIdsAuthoritative === true) { - return params.next; - } - if (runtimeExternalProfileIds.size === 0) { - return params.next; - } - const merged = cloneAuthProfileStore(params.next); - const mergedRuntimeExternalProfileIds = new Set(merged.runtimeExternalProfileIds ?? []); - const backfilledRuntimeExternalProfileIds = new Set(); - for (const profileId of runtimeExternalProfileIds) { - const existingCredential = params.existing.profiles[profileId]; - const nextCredential = merged.profiles[profileId]; - if (nextCredential) { - if ( - mergedRuntimeExternalProfileIds.has(profileId) || - (existingCredential && isDeepStrictEqual(nextCredential, existingCredential)) - ) { - mergedRuntimeExternalProfileIds.add(profileId); - } - continue; - } - if (!existingCredential) { - continue; - } - merged.profiles[profileId] = existingCredential; - mergedRuntimeExternalProfileIds.add(profileId); - backfilledRuntimeExternalProfileIds.add(profileId); - if (params.existing.usageStats?.[profileId]) { - merged.usageStats = { - ...merged.usageStats, - [profileId]: params.existing.usageStats[profileId], - }; - } - } - for (const [provider, profileIds] of Object.entries(params.existing.order ?? {})) { - const externalProfileIds = profileIds.filter((profileId) => - backfilledRuntimeExternalProfileIds.has(profileId), - ); - if (externalProfileIds.length === 0) { - continue; - } - if (merged.order?.[provider]) { - continue; - } - const existingOrder = merged.order?.[provider] ?? []; - merged.order = { - ...merged.order, - [provider]: [ - ...externalProfileIds, - ...existingOrder.filter((profileId) => !externalProfileIds.includes(profileId)), - ], - }; - } - for (const [provider, profileId] of Object.entries(params.existing.lastGood ?? {})) { - if (!backfilledRuntimeExternalProfileIds.has(profileId) || merged.lastGood?.[provider]) { - continue; - } - merged.lastGood = { - ...merged.lastGood, - [provider]: profileId, - }; - } - setRuntimeExternalProfileMetadata({ - store: merged, - profileIds: mergedRuntimeExternalProfileIds, - authoritative: params.existing.runtimeExternalProfileIdsAuthoritative === true, - }); - return merged; + setRuntimeExternalCliProfileIds( + params.store, + getRuntimeExternalCliProfileIds(params.store).filter((profileId) => + params.profileIds.has(profileId), + ), + ); } export function preserveResolvedSecretBackedCredentials(params: { @@ -844,6 +786,10 @@ function mergeRuntimeExternalProfileState(params: { } const merged = cloneAuthProfileStore(params.next); const mergedRuntimeProfileIds = new Set(merged.runtimeExternalProfileIds ?? []); + const existingRuntimeExternalCliProfileIds = new Set( + getRuntimeExternalCliProfileIds(params.existing), + ); + const mergedRuntimeExternalCliProfileIds = new Set(getRuntimeExternalCliProfileIds(merged)); const activeRuntimeProfileIds = new Set(); const nextRuntimeProfileIdsAuthoritative = params.next.runtimeExternalProfileIdsAuthoritative === true; @@ -863,12 +809,18 @@ function mergeRuntimeExternalProfileState(params: { ) { mergedRuntimeProfileIds.add(profileId); activeRuntimeProfileIds.add(profileId); + if (existingRuntimeExternalCliProfileIds.has(profileId)) { + mergedRuntimeExternalCliProfileIds.add(profileId); + } } continue; } merged.profiles[profileId] = existingCredential; mergedRuntimeProfileIds.add(profileId); activeRuntimeProfileIds.add(profileId); + if (existingRuntimeExternalCliProfileIds.has(profileId)) { + mergedRuntimeExternalCliProfileIds.add(profileId); + } } if (activeRuntimeProfileIds.size === 0) { return params.next; @@ -907,6 +859,7 @@ function mergeRuntimeExternalProfileState(params: { profileIds: mergedRuntimeProfileIds, authoritative: params.existing.runtimeExternalProfileIdsAuthoritative === true, }); + setRuntimeExternalCliProfileIds(merged, mergedRuntimeExternalCliProfileIds); return merged; } diff --git a/src/agents/auth-profiles/types.ts b/src/agents/auth-profiles/types.ts index 523a6b6e8d35..1c9a17984fa9 100644 --- a/src/agents/auth-profiles/types.ts +++ b/src/agents/auth-profiles/types.ts @@ -155,6 +155,8 @@ export type AuthProfileStore = AuthProfileSecretsStore & /** Internal effective-store ownership metadata; never exposed through the plugin SDK. */ export type RuntimeAuthProfileStore = AuthProfileStore & { + /** Runtime-only built-in CLI winners; internal provenance, never exposed or persisted. */ + runtimeExternalCliProfileIds?: string[]; runtimeLocalProfileIds?: string[]; runtimeInheritsMainState?: boolean; }; diff --git a/src/agents/model-catalog-browse.test.ts b/src/agents/model-catalog-browse.test.ts index fefd1f17e6b3..51f631088309 100644 --- a/src/agents/model-catalog-browse.test.ts +++ b/src/agents/model-catalog-browse.test.ts @@ -101,6 +101,40 @@ describe("loadPreparedModelCatalogSnapshotForBrowse", () => { expect(loadCatalog).toHaveBeenCalledExactlyOnceWith({ readOnly: false }); }); + it("keeps automatic wildcard reads on prepared facts", async () => { + const loadCatalog = vi.fn(async ({ readOnly }: { readOnly: boolean }) => + readOnly ? readOnlyCatalog : fullCatalog, + ); + + await expect( + loadPreparedModelCatalogSnapshotForBrowse({ + cfg: config({ providerWildcard: true }), + view: "configured", + preparedOnly: true, + loadCatalog, + }), + ).resolves.toBe(readOnlyCatalog); + + expect(loadCatalog).toHaveBeenCalledExactlyOnceWith({ readOnly: true }); + }); + + it("forwards explicit refresh to full discovery", async () => { + const loadCatalog = vi.fn(async ({ readOnly }: { readOnly: boolean }) => + readOnly ? readOnlyCatalog : fullCatalog, + ); + + await expect( + loadPreparedModelCatalogSnapshotForBrowse({ + cfg: config(), + view: "all", + refresh: true, + loadCatalog, + }), + ).resolves.toBe(fullCatalog); + + expect(loadCatalog).toHaveBeenCalledExactlyOnceWith({ readOnly: false, refresh: true }); + }); + it("uses the read-only catalog for provider-config views without picker allowlists", async () => { const loadCatalog = vi.fn(async ({ readOnly }: { readOnly: boolean }) => readOnly ? readOnlyCatalog : fullCatalog, diff --git a/src/agents/model-catalog-browse.ts b/src/agents/model-catalog-browse.ts index b0a5d21f390c..1c15c2be24e5 100644 --- a/src/agents/model-catalog-browse.ts +++ b/src/agents/model-catalog-browse.ts @@ -75,18 +75,22 @@ async function loadCatalogForBrowse(params: { cfg: OpenClawConfig; agentId?: string; view?: ModelCatalogBrowseView; - loadCatalog: (params: { readOnly: boolean }) => Promise; + preparedOnly?: boolean; + refresh?: boolean; + loadCatalog: (params: { readOnly: boolean; refresh?: boolean }) => Promise; empty: T; timeoutFullDiscovery?: boolean; timeoutMs?: number; onTimeout?: (timeoutMs: number) => void; }): Promise { const view = params.view ?? "default"; - const requiresFullDiscovery = modelCatalogBrowseRequiresFullDiscovery({ - cfg: params.cfg, - agentId: params.agentId, - view, - }); + const requiresFullDiscovery = + params.preparedOnly !== true && + modelCatalogBrowseRequiresFullDiscovery({ + cfg: params.cfg, + agentId: params.agentId, + view, + }); // Provider-policy wildcards newly escalate ordinary inventory views to live discovery. // Keep those implicit loads within the browse deadline; explicit all/configured loads retain // their existing completion semantics unless the caller requests a timeout. @@ -94,12 +98,18 @@ async function loadCatalogForBrowse(params: { params.timeoutFullDiscovery || (requiresFullDiscovery && (view === "default" || view === "provider-config")); if (requiresFullDiscovery && !shouldTimeoutFullDiscovery) { - return await params.loadCatalog({ readOnly: false }); + return await params.loadCatalog({ + readOnly: false, + ...(params.refresh ? { refresh: true } : {}), + }); } let timeout: NodeJS.Timeout | undefined; const timeoutMs = resolveModelCatalogBrowseTimeoutMs(params.timeoutMs); - const catalogPromise = params.loadCatalog({ readOnly: !requiresFullDiscovery }); + const catalogPromise = params.loadCatalog({ + readOnly: !requiresFullDiscovery, + ...(requiresFullDiscovery && params.refresh ? { refresh: true } : {}), + }); const catalogResult = catalogPromise.then((value) => ({ kind: "catalog" as const, value })); const timeoutPromise = new Promise<{ kind: "timeout" }>((resolve) => { timeout = globalThis.setTimeout(() => resolve({ kind: "timeout" }), timeoutMs); @@ -127,7 +137,11 @@ export function loadPreparedModelCatalogSnapshotForBrowse(params: { cfg: OpenClawConfig; agentId?: string; view?: ModelCatalogBrowseView; - loadCatalog: (params: { readOnly: boolean }) => Promise; + /** Never starts provider discovery; a completed generation cache may still be reused. */ + preparedOnly?: boolean; + /** Replaces the completed generation cache when discovery is otherwise required. */ + refresh?: boolean; + loadCatalog: (params: { readOnly: boolean; refresh?: boolean }) => Promise; timeoutFullDiscovery?: boolean; timeoutMs?: number; onTimeout?: (timeoutMs: number) => void; diff --git a/src/agents/prepared-model-catalog-worker.integration.test.ts b/src/agents/prepared-model-catalog-worker.integration.test.ts index e0179bcf3674..9e60b6014171 100644 --- a/src/agents/prepared-model-catalog-worker.integration.test.ts +++ b/src/agents/prepared-model-catalog-worker.integration.test.ts @@ -11,11 +11,13 @@ import { loadPreparedGatewayModelCatalogSnapshot, } from "../gateway/server-model-catalog.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { OPENAI_CODEX_DEFAULT_PROFILE_ID } from "./auth-profiles/constants.js"; +import { getRuntimeExternalCliProfileIds } from "./auth-profiles/runtime-external-profile-references.js"; import { clearRuntimeAuthProfileStoreSnapshots, replaceRuntimeAuthProfileStoreSnapshots, } from "./auth-profiles/runtime-snapshots.js"; -import { saveAuthProfileStore } from "./auth-profiles/store.js"; +import { ensureAuthProfileStore, saveAuthProfileStore } from "./auth-profiles/store.js"; import { encodePluginModelCatalogRelativePath, PLUGIN_MODEL_CATALOG_GENERATED_BY, @@ -47,10 +49,6 @@ const REF_ONLY_TOKEN_PROVIDER_ID = `${PROVIDER_ID}-ref-token`; const REF_ONLY_TOKEN_ENV = "OPENCLAW_WORKER_REF_ONLY_TOKEN"; const DURABLE_AUTH_PROVIDER_ID = `${PROVIDER_ID}-durable-auth`; const DURABLE_AUTH_KEY = "post-startup-durable-key-not-real"; -const WORKSPACE_AUTH_PLUGIN_ID = `${PLUGIN_ID}-workspace-auth`; -const WORKSPACE_AUTH_PROVIDER_ID = `${PROVIDER_ID}-workspace-auth`; -const WORKSPACE_AUTH_PROFILE_ID = `${WORKSPACE_AUTH_PROVIDER_ID}:workspace`; -const WORKSPACE_AUTH_KEY = "workspace-external-auth-key-not-real"; const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { afterEach(() => { clearRuntimeAuthProfileStoreSnapshots(); @@ -59,11 +57,30 @@ const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { }); }); -function createJwtWithExp(exp: number): string { - const payload = Buffer.from(JSON.stringify({ exp })).toString("base64url"); +function createJwtWithExp(exp: number, marker?: string): string { + const payload = Buffer.from(JSON.stringify({ exp, ...(marker ? { marker } : {}) })).toString( + "base64url", + ); return `header.${payload}.signature`; } +function writeCodexAuth(codexHome: string, marker: string): void { + const authPath = path.join(codexHome, "auth.json"); + fs.writeFileSync( + authPath, + JSON.stringify({ + auth_mode: "chatgpt", + tokens: { + access_token: createJwtWithExp(Math.floor(Date.now() / 1000) + 3600, marker), + refresh_token: `refresh-${marker}-not-real`, + }, + }), + "utf8", + ); + const future = new Date(Date.now() + 2_000); + fs.utimesSync(authPath, future, future); +} + function writeFixturePlugin(params: { root: string; spinMs: number }): string { const pluginDir = path.join(params.root, "plugin"); fs.mkdirSync(pluginDir, { recursive: true }); @@ -139,57 +156,11 @@ module.exports = { return pluginFile; } -function writeWorkspaceExternalAuthPlugin(workspaceDir: string): void { - const pluginDir = path.join(workspaceDir, ".openclaw", "extensions", WORKSPACE_AUTH_PLUGIN_ID); - fs.mkdirSync(pluginDir, { recursive: true }); - fs.writeFileSync( - path.join(pluginDir, "package.json"), - JSON.stringify({ - name: `@openclaw/${WORKSPACE_AUTH_PLUGIN_ID}`, - version: "1.0.0", - openclaw: { extensions: ["./index.cjs"] }, - }), - "utf8", - ); - fs.writeFileSync( - path.join(pluginDir, "openclaw.plugin.json"), - JSON.stringify({ - id: WORKSPACE_AUTH_PLUGIN_ID, - providers: [WORKSPACE_AUTH_PROVIDER_ID], - contracts: { externalAuthProviders: [WORKSPACE_AUTH_PROVIDER_ID] }, - configSchema: { type: "object", additionalProperties: false, properties: {} }, - }), - "utf8", - ); - fs.writeFileSync( - path.join(pluginDir, "index.cjs"), - `module.exports = { - id: ${JSON.stringify(WORKSPACE_AUTH_PLUGIN_ID)}, - register(api) { - api.registerProvider({ - id: ${JSON.stringify(WORKSPACE_AUTH_PROVIDER_ID)}, - label: "Workspace external auth fixture", - auth: [], - resolveExternalAuthProfiles() { - return [{ - profileId: ${JSON.stringify(WORKSPACE_AUTH_PROFILE_ID)}, - credential: { - type: "api_key", - provider: ${JSON.stringify(WORKSPACE_AUTH_PROVIDER_ID)}, - key: ${JSON.stringify(WORKSPACE_AUTH_KEY)}, - }, - persistence: "runtime-only", - }]; - }, - }); - }, -}; -`, - "utf8", - ); -} - -async function createStaticSnapshot(spinMs: number, envOverride: NodeJS.ProcessEnv = {}) { +async function createStaticSnapshot( + spinMs: number, + envOverride: NodeJS.ProcessEnv = {}, + options?: { hydrateExternalCliProviderIds?: readonly string[] }, +) { const root = tempDirs.make("openclaw-model-catalog-worker-"); const stateDir = path.join(root, "state"); const agentDir = path.join(stateDir, "agents", "main", "agent"); @@ -198,7 +169,6 @@ async function createStaticSnapshot(spinMs: number, envOverride: NodeJS.ProcessE fs.mkdirSync(agentDir, { recursive: true }); fs.mkdirSync(workspaceDir, { recursive: true }); const pluginFile = writeFixturePlugin({ root, spinMs }); - writeWorkspaceExternalAuthPlugin(workspaceDir); const env = { ...process.env, OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1", @@ -211,7 +181,7 @@ async function createStaticSnapshot(spinMs: number, envOverride: NodeJS.ProcessE const config = { agents: { defaults: { model: `${PROVIDER_ID}/sqlite-model` } }, plugins: { - allow: [PLUGIN_ID, WORKSPACE_AUTH_PLUGIN_ID], + allow: [PLUGIN_ID], load: { paths: [pluginFile] }, entries: { [PLUGIN_ID]: { enabled: true } }, }, @@ -239,6 +209,15 @@ async function createStaticSnapshot(spinMs: number, envOverride: NodeJS.ProcessE }, }, ]); + const hydratedAuthStore = options?.hydrateExternalCliProviderIds + ? ensureAuthProfileStore(agentDir, { + allowKeychainPrompt: false, + config, + externalCliProviderIds: options.hydrateExternalCliProviderIds, + readOnly: true, + syncExternalCli: false, + }) + : undefined; replacePersistedPluginModelCatalogs({ agentDir, pluginCatalogWrites: { @@ -268,6 +247,7 @@ async function createStaticSnapshot(spinMs: number, envOverride: NodeJS.ProcessE config, env, marker, + hydratedAuthStore, pluginMetadataSnapshot: build.pluginGeneration.pluginMetadataSnapshot, snapshot: build.snapshot, supersede: () => (current = false), @@ -352,32 +332,6 @@ describe("prepared model catalog worker boundary", () => { }); }); - it("refreshes workspace plugin external auth in both worker operations", async () => { - const fixture = await createStaticSnapshot(0); - - const refreshed = await loadPreparedModelRuntimeAuth(fixture.snapshot, [ - WORKSPACE_AUTH_PROVIDER_ID, - ]); - expect(refreshed).toMatchObject({ - authModes: { [WORKSPACE_AUTH_PROVIDER_ID]: "api_key" }, - authStore: { - profiles: { - [WORKSPACE_AUTH_PROFILE_ID]: expect.objectContaining({ key: WORKSPACE_AUTH_KEY }), - }, - }, - }); - - const catalog = await fixture.snapshot.loadFullModelCatalog?.(); - expect(getPreparedModelFullCatalogAuth(catalog!)).toMatchObject({ - authModes: { [WORKSPACE_AUTH_PROVIDER_ID]: "api_key" }, - authStore: { - profiles: { - [WORKSPACE_AUTH_PROFILE_ID]: expect.objectContaining({ key: WORKSPACE_AUTH_KEY }), - }, - }, - }); - }); - it("refreshes durable auth profiles added, updated, and removed after startup", async () => { const fixture = await createStaticSnapshot(0); const route = { @@ -407,7 +361,7 @@ describe("prepared model catalog worker boundary", () => { modelCatalog: { entries: [route], routeVariants: [route] }, }); const project = async () => { - const fullCatalog = await fixture.snapshot.loadFullModelCatalog?.(); + const fullCatalog = await fixture.snapshot.loadFullModelCatalog?.({ refresh: true }); const fullAuth = fullCatalog && getPreparedModelFullCatalogAuth(fullCatalog); if (!fullAuth) { throw new Error("full catalog omitted prepared auth"); @@ -558,7 +512,7 @@ describe("prepared model catalog worker boundary", () => { loadGatewayModelCatalogSnapshot: loadSnapshot, logGateway: { debug: () => undefined }, } as unknown as GatewayRequestContext; - return await buildModelsListResult({ context, params: { view: "all" } }); + return await buildModelsListResult({ context, params: { view: "all", refresh: true } }); }; await expect(listModels()).resolves.toMatchObject({ @@ -585,7 +539,52 @@ describe("prepared model catalog worker boundary", () => { }); }); - it("shares in-flight discovery but reruns completed refreshes with prepared auth and SQLite facts", async () => { + it("refreshes and removes a Codex login that existed in the prepared generation", async () => { + const codexHome = tempDirs.make("openclaw-prepared-codex-"); + writeCodexAuth(codexHome, "startup"); + const previousCodexHome = process.env.CODEX_HOME; + process.env.CODEX_HOME = codexHome; + let fixture: Awaited>; + try { + fixture = await createStaticSnapshot(0, {}, { hydrateExternalCliProviderIds: ["openai"] }); + } finally { + if (previousCodexHome === undefined) { + delete process.env.CODEX_HOME; + } else { + process.env.CODEX_HOME = previousCodexHome; + } + } + const preparedStore = getPreparedModelRuntimeAuthStore(fixture.snapshot); + expect(fixture.hydratedAuthStore?.profiles[OPENAI_CODEX_DEFAULT_PROFILE_ID]).toMatchObject({ + type: "oauth", + refresh: "refresh-startup-not-real", + }); + expect(preparedStore?.profiles[OPENAI_CODEX_DEFAULT_PROFILE_ID]).toMatchObject({ + type: "oauth", + refresh: "refresh-startup-not-real", + }); + expect(preparedStore && getRuntimeExternalCliProfileIds(preparedStore)).toEqual([ + OPENAI_CODEX_DEFAULT_PROFILE_ID, + ]); + + writeCodexAuth(codexHome, "rotated"); + const rotated = await loadPreparedModelRuntimeAuth(fixture.snapshot, { + providerIds: [], + profileIds: [OPENAI_CODEX_DEFAULT_PROFILE_ID], + }); + expect(rotated?.authStore.profiles[OPENAI_CODEX_DEFAULT_PROFILE_ID]).toMatchObject({ + type: "oauth", + refresh: "refresh-rotated-not-real", + }); + + fs.rmSync(path.join(codexHome, "auth.json")); + const loggedOut = await loadPreparedModelRuntimeAuth(fixture.snapshot, { + providerIds: ["openai"], + }); + expect(loggedOut?.authStore.profiles[OPENAI_CODEX_DEFAULT_PROFILE_ID]).toBeUndefined(); + }); + + it("shares in-flight discovery, caches completion, and explicitly refreshes prepared facts", async () => { const fixture = await createStaticSnapshot(750); let settled = false; const first = fixture.snapshot.loadFullModelCatalog?.().finally(() => { @@ -603,7 +602,8 @@ describe("prepared model catalog worker boundary", () => { id: "proof-refresh-1-sqlite-true-shared-true-unrelated-true", }), ); - await expect(fixture.snapshot.loadFullModelCatalog?.()).resolves.toEqual( + await expect(fixture.snapshot.loadFullModelCatalog?.()).resolves.toBe(catalog); + await expect(fixture.snapshot.loadFullModelCatalog?.({ refresh: true })).resolves.toEqual( expect.objectContaining({ entries: expect.arrayContaining([ expect.objectContaining({ diff --git a/src/agents/prepared-model-catalog-worker.test.ts b/src/agents/prepared-model-catalog-worker.test.ts index 844981169b93..c62dc54613de 100644 --- a/src/agents/prepared-model-catalog-worker.test.ts +++ b/src/agents/prepared-model-catalog-worker.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it, vi } from "vitest"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js"; -import { createPreparedModelCatalogWorkerInput } from "./prepared-model-catalog-worker.js"; +import { + createPreparedModelAuthRefreshWorkerInput, + createPreparedModelCatalogWorkerInput, +} from "./prepared-model-catalog-worker.js"; import type { PreparedModelRuntimeAgentFacts } from "./prepared-model-runtime.facts.js"; vi.mock("../plugins/manifest-registry-installed.js", () => ({ @@ -8,6 +11,27 @@ vi.mock("../plugins/manifest-registry-installed.js", () => ({ })); describe("prepared model catalog worker input", () => { + it("serializes and fingerprints auth refresh profile scope", () => { + const params = { + agentDir: "/tmp/agent", + authStore: { version: 1, profiles: {} }, + config: {}, + env: {}, + providerIds: ["openai"], + }; + const scoped = createPreparedModelAuthRefreshWorkerInput({ + ...params, + profileIds: ["openai:work", "openai:default", "openai:work"], + }); + const other = createPreparedModelAuthRefreshWorkerInput({ + ...params, + profileIds: ["openai:default"], + }); + + expect(structuredClone(scoped).profileIds).toEqual(["openai:default", "openai:work"]); + expect(scoped.generationFingerprint).not.toBe(other.generationFingerprint); + }); + it("preserves SecretRef identity beside materialized literals", () => { const authStore = { version: 1, diff --git a/src/agents/prepared-model-catalog-worker.ts b/src/agents/prepared-model-catalog-worker.ts index 94828d53312c..d8d2be72ccd3 100644 --- a/src/agents/prepared-model-catalog-worker.ts +++ b/src/agents/prepared-model-catalog-worker.ts @@ -30,10 +30,10 @@ export type PreparedModelAuthRefreshWorkerInput = Readonly<{ generationFingerprint: string; agentDir: string; inheritedAuthDir?: string; - workspaceDir?: string; authStore: AuthProfileStore; config: PreparedModelRuntimeInput["config"]; env: NodeJS.ProcessEnv; + profileIds?: readonly string[]; providerIds: readonly string[]; }>; @@ -140,10 +140,10 @@ export function createPreparedModelCatalogWorkerInput(params: { export function createPreparedModelAuthRefreshWorkerInput(params: { agentDir: string; inheritedAuthDir?: string; - workspaceDir?: string; authStore: AuthProfileStore; config: PreparedModelRuntimeInput["config"]; env: NodeJS.ProcessEnv; + profileIds?: readonly string[]; providerIds: readonly string[]; }): PreparedModelAuthRefreshWorkerInput { const providerIds = [...new Set(params.providerIds)].toSorted((left, right) => @@ -151,23 +151,26 @@ export function createPreparedModelAuthRefreshWorkerInput(params: { ); const authStore = cloneAuthProfileStore(params.authStore); const env = { ...params.env }; + const profileIds = params.profileIds + ? [...new Set(params.profileIds)].toSorted((left, right) => left.localeCompare(right)) + : undefined; return { kind: "auth-refresh", agentDir: params.agentDir, ...(params.inheritedAuthDir ? { inheritedAuthDir: params.inheritedAuthDir } : {}), - ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), generationFingerprint: fingerprintPreparedRuntimeFacts({ agentDir: params.agentDir, inheritedAuthDir: params.inheritedAuthDir, - workspaceDir: params.workspaceDir, authStore, config: params.config, env, + profileIds, providerIds, }), authStore, config: params.config, env, + ...(profileIds ? { profileIds } : {}), providerIds, }; } @@ -230,7 +233,11 @@ function runPreparedModelWorker(params: { worker.removeAllListeners(); const finish = () => { if (outcome.status === "resolved") { - resolve(outcome.value); + if (params.isCurrent()) { + resolve(outcome.value); + } else { + reject(superseded()); + } } else { reject(outcome.error); } diff --git a/src/agents/prepared-model-catalog.ts b/src/agents/prepared-model-catalog.ts index fa632efe054c..590e9204ff68 100644 --- a/src/agents/prepared-model-catalog.ts +++ b/src/agents/prepared-model-catalog.ts @@ -16,7 +16,9 @@ import { PreparedModelCatalogConfigReplacedError } from "./prepared-model-catalo import type { ResolvedPublishedModelCatalogOwner } from "./prepared-model-catalog.types.js"; import { getPreparedModelRuntimeAuthMaterializations, + loadPreparedModelRuntimeAuth, setPreparedModelRuntimeAuthMaterializations, + setPreparedModelRuntimeAuthLoader, setPreparedModelRuntimeAuthStore, } from "./prepared-model-runtime-auth.js"; import { isPreparedModelCatalogFull } from "./prepared-model-runtime.facts.js"; @@ -48,6 +50,8 @@ export type LoadPreparedModelCatalogParams = { workspaceDir?: string; env?: NodeJS.ProcessEnv; providerDiscoveryProviderIds?: readonly string[]; + /** Rebuilds a completed full catalog instead of reusing this generation's cache. */ + refreshFullCatalog?: boolean; /** Scoped read-only loads may run live discovery for the scoped providers only. */ scopedLiveProviderDiscovery?: boolean; allowGatewaySubagentBinding?: boolean; @@ -63,11 +67,18 @@ type PreparedModelCatalogConfigPolicy = "exact" | "published"; async function materializeRequestedModelCatalog( snapshot: PreparedModelRuntimeSnapshot, readOnly: boolean | undefined, + refreshFullCatalog: boolean | undefined, ): Promise { - if (readOnly === true || !snapshot.loadFullModelCatalog) { + if (!snapshot.loadFullModelCatalog) { + return snapshot; + } + const modelCatalog = + readOnly === true + ? snapshot.readFullModelCatalog?.() + : await snapshot.loadFullModelCatalog({ refresh: refreshFullCatalog === true }); + if (!modelCatalog) { return snapshot; } - const modelCatalog = await snapshot.loadFullModelCatalog(); const fullAuth = getPreparedModelFullCatalogAuth(modelCatalog); if (!fullAuth) { throw new Error("prepared full model catalog omitted its auth generation"); @@ -77,9 +88,13 @@ async function materializeRequestedModelCatalog( authModes: fullAuth.authModes, modelCatalog, }); - // A materialized full read owns the worker's refreshed auth generation. Do not copy the - // original loader or Gateway projection would start a second, split refresh after discovery. setPreparedModelRuntimeAuthStore(materialized, fullAuth.authStore); + // Later explicit auth refreshes stay bound to the original owner generation. Ordinary reads + // consume the full worker's paired auth without invoking this loader. + setPreparedModelRuntimeAuthLoader( + materialized, + async (scope) => (await loadPreparedModelRuntimeAuth(snapshot, scope)) ?? fullAuth, + ); setPreparedModelRuntimeAuthMaterializations( materialized, getPreparedModelRuntimeAuthMaterializations(snapshot), @@ -288,6 +303,7 @@ async function loadPreparedModelCatalogOwnerSnapshotWithPolicy( return await materializeRequestedModelCatalog( await resolvePreparedModelCatalogOwnerSnapshotWithPolicy(params, configPolicy), params.readOnly, + params.refreshFullCatalog, ); } diff --git a/src/agents/prepared-model-catalog.worker.ts b/src/agents/prepared-model-catalog.worker.ts index 73aea3182e86..6108cf3555d6 100644 --- a/src/agents/prepared-model-catalog.worker.ts +++ b/src/agents/prepared-model-catalog.worker.ts @@ -6,8 +6,9 @@ import { resolveUsableAgentCredentialModes, } from "./agent-auth-credentials.js"; import { resolveAmbientAgentCredentialsForDiscovery } from "./agent-auth-discovery.js"; -import { overlayExternalAuthProfiles } from "./auth-profiles/external-auth.js"; +import { overlayExternalCliAuthProfiles } from "./auth-profiles/external-auth.js"; import { listExternalCliSyncProviderIds } from "./auth-profiles/external-cli-sync.js"; +import { mergeRuntimeExternalProfileReferences } from "./auth-profiles/runtime-external-profile-references.js"; import { replaceRuntimeAuthProfileStoreSnapshots } from "./auth-profiles/runtime-snapshots.js"; import { loadAuthProfileStoreWithoutExternalProfiles, @@ -27,7 +28,7 @@ function refreshAuthStore(params: { authStore: PreparedModelCatalogWorkerInput["authStore"]; config: PreparedModelCatalogWorkerInput["input"]["config"]; env: NodeJS.ProcessEnv; - workspaceDir?: string; + profileIds?: readonly string[]; providerIds?: readonly string[]; }) { const durable = preserveResolvedSecretBackedCredentials({ @@ -48,12 +49,15 @@ function refreshAuthStore(params: { durable.profiles[profileId] = credential; } } - return overlayExternalAuthProfiles(durable, { - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, + const prepared = mergeRuntimeExternalProfileReferences({ + next: durable, + existing: params.authStore, + }); + return overlayExternalCliAuthProfiles(prepared, { config: params.config, env: params.env, ...(params.providerIds ? { externalCliProviderIds: params.providerIds } : {}), + ...(params.profileIds ? { externalCliProfileIds: params.profileIds } : {}), allowKeychainPrompt: false, }); } @@ -66,10 +70,10 @@ export async function runPreparedModelCatalogWorkerInput( const authStore = refreshAuthStore({ agentDir: value.agentDir, inheritedAuthDir: value.inheritedAuthDir, - workspaceDir: value.workspaceDir, authStore: value.authStore, config: value.config, env: value.env, + ...(value.profileIds ? { profileIds: value.profileIds } : {}), providerIds: value.providerIds, }); return { @@ -100,17 +104,14 @@ export async function runPreparedModelCatalogWorkerInput( } // Full discovery is one point-in-time operation: refresh first, then let every provider hook // and the returned availability projection consume the same exact store. - const authStore = withPluginRuntimeRegistryScope(prepared.pluginGeneration.pluginRegistry, () => - refreshAuthStore({ - agentDir: value.input.agentDir, - inheritedAuthDir: value.input.inheritedAuthDir, - authStore: value.authStore, - config: value.input.config, - env: value.input.env ?? process.env, - providerIds: listExternalCliSyncProviderIds(), - workspaceDir: value.input.workspaceDir, - }), - ); + const authStore = refreshAuthStore({ + agentDir: value.input.agentDir, + inheritedAuthDir: value.input.inheritedAuthDir, + authStore: value.authStore, + config: value.input.config, + env: value.input.env ?? process.env, + providerIds: listExternalCliSyncProviderIds(), + }); replaceRuntimeAuthProfileStoreSnapshots([{ agentDir: value.input.agentDir, store: authStore }]); const ambientCredentials = withPluginRuntimeRegistryScope( prepared.pluginGeneration.pluginRegistry, diff --git a/src/agents/prepared-model-runtime-auth.ts b/src/agents/prepared-model-runtime-auth.ts index 1e8e3057dc85..d0d60b0e94de 100644 --- a/src/agents/prepared-model-runtime-auth.ts +++ b/src/agents/prepared-model-runtime-auth.ts @@ -7,12 +7,17 @@ export type PreparedModelRuntimeAuth = Readonly<{ authModes: PreparedAgentCredentialModes; }>; +export type PreparedModelRuntimeAuthScope = Readonly<{ + providerIds: readonly string[]; + profileIds?: readonly string[]; +}>; + /** Private auth facts owned by an immutable prepared model generation. */ const authStoreBySnapshot = new WeakMap(); const materializationsBySnapshot = new WeakMap(); const authLoaderBySnapshot = new WeakMap< object, - (providerIds: readonly string[]) => Promise + (scope: PreparedModelRuntimeAuthScope) => Promise >(); // Secret-bearing state stays lifecycle-owned without becoming part of the public snapshot shape. @@ -29,18 +34,18 @@ export function getPreparedModelRuntimeAuthStore(snapshot: object): AuthProfileS export function setPreparedModelRuntimeAuthLoader( snapshot: object, - loader: (providerIds: readonly string[]) => Promise, + loader: (scope: PreparedModelRuntimeAuthScope) => Promise, ): void { authLoaderBySnapshot.set(snapshot, loader); } export async function loadPreparedModelRuntimeAuth( snapshot: object & { authModes?: PreparedAgentCredentialModes }, - providerIds: readonly string[], + scope: PreparedModelRuntimeAuthScope, ): Promise { const loader = authLoaderBySnapshot.get(snapshot); if (loader) { - return await loader(providerIds); + return await loader(scope); } const authStore = authStoreBySnapshot.get(snapshot); return authStore ? { authStore, authModes: snapshot.authModes ?? {} } : undefined; diff --git a/src/agents/prepared-model-runtime.build.ts b/src/agents/prepared-model-runtime.build.ts index f5089c9b2d9f..3b03f3c15448 100644 --- a/src/agents/prepared-model-runtime.build.ts +++ b/src/agents/prepared-model-runtime.build.ts @@ -18,6 +18,7 @@ import { setPreparedModelRuntimeAuthLoader, setPreparedModelRuntimeAuthStore, type PreparedModelRuntimeAuth, + type PreparedModelRuntimeAuthScope, } from "./prepared-model-runtime-auth.js"; import { PreparedModelRuntimePublicationSupersededError } from "./prepared-model-runtime.errors.js"; import { @@ -49,8 +50,9 @@ const MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS = 1; const limitFullModelCatalogBuild = pLimit(MAX_CONCURRENT_FULL_MODEL_CATALOG_BUILDS); type PreparedModelRuntimeCatalogAccess = Readonly<{ - loadFullModelCatalog: () => Promise; - loadAuth: (providerIds: readonly string[]) => Promise; + readFullModelCatalog: () => ModelCatalogSnapshot | undefined; + loadFullModelCatalog: (options?: { refresh?: boolean }) => Promise; + loadAuth: (scope: PreparedModelRuntimeAuthScope) => Promise; }>; type PreparedModelRuntimeBuildGuards = | ReadonlyMap boolean> @@ -119,8 +121,9 @@ function createFullModelCatalogAccess(params: { agentBuildCompletions: Map>; isCurrent: () => boolean; }): PreparedModelRuntimeCatalogAccess { - // Concurrent readers share discovery, but completed results are discarded so - // refreshable providers can publish changed inventory on the next explicit read. + // The completed catalog is generation-owned. Explicit refresh replaces it only after a + // successful build, so failed refreshes cannot discard the last verified inventory. + let fullCatalog: ModelCatalogSnapshot | undefined; let pending: Promise | undefined; let pendingAuth: | { @@ -136,27 +139,25 @@ function createFullModelCatalogAccess(params: { } }; return { - loadAuth: (providerIds) => { + loadAuth: ({ providerIds, profileIds }) => { const key = [...new Set(providerIds)] .toSorted((left, right) => left.localeCompare(right)) .join("\0"); - if (key.length === 0) { - return Promise.resolve({ - authStore: params.agentFacts.authStore, - authModes: resolveUsableAgentCredentialModes(params.agentFacts.credentials), - }); - } - if (pendingAuth?.key === key) { + const profileKey = [...new Set(profileIds ?? [])] + .toSorted((left, right) => left.localeCompare(right)) + .join("\0"); + const cacheKey = `${key}\0\0${profileKey}`; + if (pendingAuth?.key === cacheKey) { return pendingAuth.promise; } const input = createPreparedModelAuthRefreshWorkerInput({ agentDir: params.agentFacts.input.agentDir, inheritedAuthDir: params.agentFacts.input.inheritedAuthDir, - workspaceDir: params.agentFacts.input.workspaceDir, authStore: params.agentFacts.authStore, config: params.agentFacts.input.config, env: params.agentFacts.env, providerIds, + ...(profileIds?.length ? { profileIds } : {}), }); const promise = runPreparedModelAuthRefreshWorker({ input, @@ -177,12 +178,24 @@ function createFullModelCatalogAccess(params: { pendingAuth = undefined; } }); - pendingAuth = { key, promise }; + pendingAuth = { key: cacheKey, promise }; return promise; }, - loadFullModelCatalog: () => { + readFullModelCatalog: () => { + assertCurrent(); + return fullCatalog; + }, + loadFullModelCatalog: (options) => { + try { + assertCurrent(); + } catch (error) { + return Promise.reject(error); + } + if (!options?.refresh && fullCatalog) { + return Promise.resolve(fullCatalog); + } if (!pending) { - pending = runSerializedPreparedModelRuntimeTask({ + const build = runSerializedPreparedModelRuntimeTask({ agentDir: params.agentFacts.input.agentDir, agentBuildCompletions: params.agentBuildCompletions, isCurrent: params.isCurrent, @@ -201,9 +214,15 @@ function createFullModelCatalogAccess(params: { assertCurrent(); return catalog; }), - }).finally(() => { - pending = undefined; }); + pending = build + .then((catalog) => { + fullCatalog = catalog; + return catalog; + }) + .finally(() => { + pending = undefined; + }); } return pending; }, @@ -241,6 +260,7 @@ function createSnapshot( ...(messageToolCatalog ? { messageToolCatalog } : {}), ...(mediaCapabilityProviders ? { mediaCapabilityProviders } : {}), modelCatalog, + readFullModelCatalog: catalogAccess.readFullModelCatalog, loadFullModelCatalog: catalogAccess.loadFullModelCatalog, configuredRuntimeModels, inlineProviderModels, diff --git a/src/agents/prepared-model-runtime.facts.ts b/src/agents/prepared-model-runtime.facts.ts index 68d6d36686e0..fb90f4831ff3 100644 --- a/src/agents/prepared-model-runtime.facts.ts +++ b/src/agents/prepared-model-runtime.facts.ts @@ -25,6 +25,7 @@ import { discoverModels, discoverModelsFromCapturedSources, } from "./agent-model-discovery.js"; +import { getPreparedRuntimeAuthProfileStoreSnapshot } from "./auth-profiles/store.js"; import type { AuthProfileStore } from "./auth-profiles/types.js"; import { buildInlineProviderModels, @@ -122,12 +123,25 @@ function prepareAgentFacts( additionalProviderIds: readonly string[] = [], ): PreparedModelRuntimeAgentBaseFacts { const env = input.env ?? process.env; + const publishedStore = getPreparedRuntimeAuthProfileStoreSnapshot( + input.agentDir, + input.inheritedAuthDir, + ); + // Runtime-only external profiles exist only in the published auth generation. Re-reading the + // durable store here would erase startup hydration before this owner can carry it forward. + const preparedStore = + publishedStore && + (publishedStore.runtimeExternalProfileIds !== undefined || + publishedStore.runtimeExternalProfileIdsAuthoritative === true) + ? publishedStore + : undefined; const authFacts = discoverAuthStorageFacts(input.agentDir, { config: input.config, // Prepared owners consume only the already-published runtime auth generation. External CLI // hydration belongs to startup/control-plane and turn-time producers, never rebuilds. readOnly: true, ambientCredentials, + ...(preparedStore ? { preparedStore } : {}), ...(input.skipCredentials ? { skipCredentials: true } : {}), ...(input.inheritedAuthDir ? { inheritedAuthDir: input.inheritedAuthDir } : {}), ...(input.workspaceDir ? { workspaceDir: input.workspaceDir } : {}), diff --git a/src/agents/prepared-model-runtime.startup-static.test.ts b/src/agents/prepared-model-runtime.startup-static.test.ts index e9053ff04875..13d8ea74d59c 100644 --- a/src/agents/prepared-model-runtime.startup-static.test.ts +++ b/src/agents/prepared-model-runtime.startup-static.test.ts @@ -168,6 +168,7 @@ vi.mock("./agent-scope.js", () => ({ })); vi.mock("./auth-profiles/runtime-snapshots.js", () => ({ + getPreparedRuntimeAuthProfileStoreSnapshotCore: () => undefined, registerRuntimeAuthProfileStoreMutationListener: ( listener: (event: { agentDir?: string; affectsInheritedStores: boolean }) => void, ) => { @@ -416,7 +417,17 @@ describe("prepared model runtime Gateway catalog mode", () => { routeVariants: [], }); expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled(); + expect(mocks.runPreparedModelCatalogWorker).toHaveBeenCalledOnce(); + expect(snapshot?.readFullModelCatalog?.()).toEqual({ entries: [], routeVariants: [] }); + + await snapshot?.loadFullModelCatalog?.({ refresh: true }); expect(mocks.runPreparedModelCatalogWorker).toHaveBeenCalledTimes(2); + mocks.runPreparedModelCatalogWorker.mockRejectedValueOnce(new Error("refresh failed")); + await expect(snapshot?.loadFullModelCatalog?.({ refresh: true })).rejects.toThrow( + "refresh failed", + ); + expect(snapshot?.readFullModelCatalog?.()).toEqual({ entries: [], routeVariants: [] }); + expect(mocks.runPreparedModelCatalogWorker).toHaveBeenCalledTimes(3); expect(mocks.prepareStaticCatalog).toHaveBeenCalledOnce(); expect(mocks.discoverModels).toHaveBeenCalledOnce(); @@ -425,7 +436,7 @@ describe("prepared model runtime Gateway catalog mode", () => { affectsInheritedStores: false, }); await expect(snapshot?.loadFullModelCatalog?.()).rejects.toThrow("superseded"); - expect(mocks.runPreparedModelCatalogWorker).toHaveBeenCalledTimes(2); + expect(mocks.runPreparedModelCatalogWorker).toHaveBeenCalledTimes(3); }); it("publishes exact dynamic configured models without building a live catalog", async () => { diff --git a/src/agents/prepared-model-runtime.types.ts b/src/agents/prepared-model-runtime.types.ts index 33038c41ce07..523ac47fba79 100644 --- a/src/agents/prepared-model-runtime.types.ts +++ b/src/agents/prepared-model-runtime.types.ts @@ -53,8 +53,10 @@ export type PreparedModelRuntimeSnapshot = Readonly<{ * Full inventory discovery is deliberately outside the startup publication boundary. */ modelCatalog: ModelCatalogSnapshot; + /** Reads a completed full catalog without starting provider discovery. */ + readFullModelCatalog?: () => ModelCatalogSnapshot | undefined; /** Builds this generation's full control-plane catalog without replacing turn facts. */ - loadFullModelCatalog?: () => Promise; + loadFullModelCatalog?: (options?: { refresh?: boolean }) => Promise; /** Full static models for configured refs, resolved once at the lifecycle boundary. */ configuredRuntimeModels: readonly PreparedConfiguredRuntimeModel[]; /** Inline provider projection prepared once for all resolutions owned by this snapshot. */ diff --git a/src/gateway/local-request-context.test.ts b/src/gateway/local-request-context.test.ts index 2ac740e118e2..caeceb6c3f2c 100644 --- a/src/gateway/local-request-context.test.ts +++ b/src/gateway/local-request-context.test.ts @@ -151,7 +151,7 @@ describe("local gateway request context", () => { const list = () => withLocalGatewayRequestScope({ deps: {} as CliDeps, getRuntimeConfig: () => cfg }, () => - dispatchGatewayMethodInProcessRaw("models.list", { view: "all" }), + dispatchGatewayMethodInProcessRaw("models.list", { view: "configured", refresh: true }), ); const loggedIn = await list(); const loggedOut = await list(); @@ -162,10 +162,14 @@ describe("local gateway request context", () => { }); expect(loggedOut).toMatchObject({ ok: true, - payload: { models: [expect.objectContaining({ id: "local-auth-model", available: false })] }, + payload: { models: [] }, + }); + expect(refreshAuth).toHaveBeenNthCalledWith(1, { + providerIds: ["local-auth-provider"], + }); + expect(refreshAuth).toHaveBeenNthCalledWith(2, { + providerIds: ["local-auth-provider"], }); - expect(refreshAuth).toHaveBeenNthCalledWith(1, ["local-auth-provider"]); - expect(refreshAuth).toHaveBeenNthCalledWith(2, ["local-auth-provider"]); loadOwner.mockRestore(); }); diff --git a/src/gateway/local-request-context.ts b/src/gateway/local-request-context.ts index fafe2baf9fbe..0f71021c4eb0 100644 --- a/src/gateway/local-request-context.ts +++ b/src/gateway/local-request-context.ts @@ -116,7 +116,6 @@ function createLocalGatewayRequestContext( loadPreparedGatewayModelCatalogSnapshot({ ...loadParams, getConfig: params.getRuntimeConfig, - refreshAuth: true, }), readPrepared: (loadParams) => readPreparedGatewayModelCatalogOwnerSnapshot({ diff --git a/src/gateway/server-kernel.ts b/src/gateway/server-kernel.ts index 9e23c2d0d947..d789c2938747 100644 --- a/src/gateway/server-kernel.ts +++ b/src/gateway/server-kernel.ts @@ -95,8 +95,7 @@ const readPreparedGatewayModelCatalogOwnerSnapshot: ReadPreparedGatewayModelCata }; registerGatewayModelCatalogPrivateAccess(loadGatewayModelCatalogSnapshot, { - loadDeferred: (params) => - loadPreparedGatewayModelCatalogSnapshot({ ...params, refreshAuth: true }), + loadDeferred: (params) => loadPreparedGatewayModelCatalogSnapshot(params), readPrepared: readPreparedGatewayModelCatalogOwnerSnapshot, }); diff --git a/src/gateway/server-methods/models-auth-status.test.ts b/src/gateway/server-methods/models-auth-status.test.ts index 5108ac488099..ebd40e42a2fc 100644 --- a/src/gateway/server-methods/models-auth-status.test.ts +++ b/src/gateway/server-methods/models-auth-status.test.ts @@ -32,11 +32,6 @@ const mocks = vi.hoisted(() => ({ agentId === "main" ? "/tmp/agent" : `/tmp/agent-${agentId}`, ), resolveDefaultAgentId: vi.fn(() => "main"), - ensureAuthProfileStore: vi.fn((agentDir?: string, options?: unknown): AuthProfileStore => { - void agentDir; - void options; - return { version: 1, profiles: {} }; - }), ensureAuthProfileStoreWithoutExternalProfiles: vi.fn((agentDir?: string): AuthProfileStore => { void agentDir; return { version: 1, profiles: {} }; @@ -49,10 +44,11 @@ const mocks = vi.hoisted(() => ({ resolvePersistedAuthProfileOwnerAgentDir: vi.fn( (params: { agentDir?: string }) => params.agentDir, ), - clearRuntimeAuthProfileStoreSnapshots: vi.fn(), refreshActiveProviderAuthRuntimeSnapshot: vi.fn(async () => false), clearCurrentProviderAuthState: vi.fn(), warmCurrentProviderAuthStateOffMainThread: vi.fn(async (_cfg: unknown) => {}), + loadDeferredCatalog: vi.fn(), + readPreparedCatalog: vi.fn(), buildAuthHealthSummary: vi.fn( (): AuthHealthSummary => ({ now: 0, warnAfterMs: 0, profiles: [], providers: [] }), ), @@ -75,14 +71,12 @@ vi.mock("../../agents/auth-profiles.js", async () => { ); return { ...actual, - ensureAuthProfileStore: mocks.ensureAuthProfileStore, ensureAuthProfileStoreWithoutExternalProfiles: mocks.ensureAuthProfileStoreWithoutExternalProfiles, listProfilesForProvider: mocks.listProfilesForProvider, removeAuthProfilesAcrossOwnerStores: mocks.removeAuthProfilesAcrossOwnerStores, removeProviderAuthProfilesWithLock: mocks.removeProviderAuthProfilesWithLock, resolvePersistedAuthProfileOwnerAgentDir: mocks.resolvePersistedAuthProfileOwnerAgentDir, - clearRuntimeAuthProfileStoreSnapshots: mocks.clearRuntimeAuthProfileStoreSnapshots, }; }); @@ -109,6 +103,11 @@ vi.mock("../../agents/model-provider-auth.js", () => ({ warmCurrentProviderAuthStateOffMainThread: mocks.warmCurrentProviderAuthStateOffMainThread, })); +vi.mock("../server-model-catalog-auth.js", () => ({ + loadDeferredCatalog: mocks.loadDeferredCatalog, + readPreparedCatalog: mocks.readPreparedCatalog, +})); + import { aggregateRefreshableAuthStatus, invalidateModelAuthStatusCache, @@ -196,6 +195,35 @@ function createLogoutOptions( } const requireRecord = createRequireRecord("record", "expected-non-array-record"); +let preparedAuthStore: AuthProfileStore = { version: 1, profiles: {} }; +let preparedMetadataSnapshot: unknown; + +function setPreparedAuthStore(store: AuthProfileStore): void { + preparedAuthStore = store; +} + +function setPreparedMetadataSnapshot(snapshot: unknown): void { + preparedMetadataSnapshot = snapshot; +} + +function createPreparedOwnerSnapshot(agentId: string) { + const config = mocks.getRuntimeConfig.mock.results.at(-1)?.value ?? {}; + const agentDir = + mocks.resolveAgentDir.mock.results.at(-1)?.value ?? + (agentId === "main" ? "/tmp/agent" : `/tmp/agent-${agentId}`); + return { + agentId, + agentDir, + workspaceDir: "/tmp/workspace", + config, + entries: [], + routeVariants: [], + authModes: {}, + authStore: preparedAuthStore, + authMaterializations: [], + metadataSnapshot: preparedMetadataSnapshot as never, + }; +} function firstRespondCall( opts: GatewayRequestHandlerOptions & { respond: ReturnType }, @@ -203,10 +231,6 @@ function firstRespondCall( return opts.respond.mock.calls[0]; } -function firstEnsureAuthProfileStoreCall() { - return mocks.ensureAuthProfileStore.mock.calls[0]; -} - function firstBuildAuthHealthSummaryCall() { return mocks.buildAuthHealthSummary.mock.calls[0] as unknown as | [{ providers?: string[]; allowKeychainPrompt?: boolean }] @@ -246,7 +270,18 @@ function resetAuthStatusMocks(): void { agentId === "main" ? "/tmp/agent" : `/tmp/agent-${agentId}`, ); mocks.resolveDefaultAgentId.mockReturnValue("main"); - mocks.ensureAuthProfileStore.mockReturnValue({ version: 1, profiles: {} }); + setPreparedAuthStore({ version: 1, profiles: {} }); + setPreparedMetadataSnapshot({ + index: { plugins: [] }, + manifestRegistry: { plugins: [] }, + plugins: [], + }); + mocks.readPreparedCatalog.mockImplementation(async (_context, agentId: string) => + createPreparedOwnerSnapshot(agentId), + ); + mocks.loadDeferredCatalog.mockImplementation(async (_context, agentId: string) => + createPreparedOwnerSnapshot(agentId), + ); mocks.ensureAuthProfileStoreWithoutExternalProfiles.mockReturnValue({ version: 1, profiles: {}, @@ -267,17 +302,20 @@ function resetAuthStatusMocks(): void { mocks.refreshActiveProviderAuthRuntimeSnapshot.mockResolvedValue(false); } +function firstDeferredAuthScope() { + expect(mocks.loadDeferredCatalog).toHaveBeenCalledTimes(1); + const [, agentId, options] = mocks.loadDeferredCatalog.mock.calls[0] ?? []; + expect(agentId).toBe("main"); + const deferredOptions = requireRecord(options); + expect(deferredOptions.readOnly).toBe(true); + expect(deferredOptions.refreshAuth).toBe(true); + return requireRecord(deferredOptions.authScope); +} + afterEach(() => { vi.unstubAllEnvs(); }); -function firstExternalCliAuthOption() { - expect(mocks.ensureAuthProfileStore).toHaveBeenCalledTimes(1); - expect(firstEnsureAuthProfileStoreCall()?.[0]).toBe("/tmp/agent"); - const [, options] = firstEnsureAuthProfileStoreCall() ?? []; - return requireRecord(requireRecord(options).externalCli); -} - function expectLogoutFailurePreservesRun(params: { opts: ReturnType; runId: string; @@ -358,10 +396,7 @@ describe("models.authStatus", () => { await handler(opts); expect(mocks.resolveAgentDir).toHaveBeenCalledWith(cfg, expectedAgentId); - expect(mocks.ensureAuthProfileStore).toHaveBeenCalledWith( - expectedAgentId === "main" ? "/tmp/agent" : "/tmp/agent-writer", - expect.any(Object), - ); + expect(mocks.readPreparedCatalog).toHaveBeenCalledWith(expect.anything(), expectedAgentId); }, ); @@ -374,7 +409,8 @@ describe("models.authStatus", () => { await handler(opts); expect(mocks.resolveAgentDir).not.toHaveBeenCalled(); - expect(mocks.ensureAuthProfileStore).not.toHaveBeenCalled(); + expect(mocks.readPreparedCatalog).not.toHaveBeenCalled(); + expect(mocks.loadDeferredCatalog).not.toHaveBeenCalled(); expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).not.toHaveBeenCalled(); const [ok, payload, error] = firstRespondCall(opts) ?? []; expect(ok).toBe(false); @@ -395,10 +431,7 @@ describe("models.authStatus", () => { await handler(opts); expect(mocks.resolveAgentDir).toHaveBeenCalledWith(cfg, "_writer"); - expect(mocks.ensureAuthProfileStore).toHaveBeenCalledWith( - "/tmp/agent-_writer", - expect.any(Object), - ); + expect(mocks.readPreparedCatalog).toHaveBeenCalledWith(expect.anything(), "_writer"); expect(firstRespondCall(opts)?.[0]).toBe(true); }); @@ -413,7 +446,7 @@ describe("models.authStatus", () => { await handler(opts); expect(mocks.resolveAgentDir).not.toHaveBeenCalled(); - expect(mocks.ensureAuthProfileStore).not.toHaveBeenCalled(); + expect(mocks.readPreparedCatalog).not.toHaveBeenCalled(); expect(firstRespondCall(opts)?.[2]).toEqual({ code: "INVALID_REQUEST", message: `unknown agent id "${agentId}"`, @@ -422,7 +455,7 @@ describe("models.authStatus", () => { }, ); - it("rebuilds fresh auth snapshots for each requested agent", async () => { + it("reads the published auth owner for each requested agent", async () => { const cfg = { agents: { list: [{ id: "main", default: true }, { id: "writer" }] } }; mocks.getRuntimeConfig.mockReturnValue(cfg); mocks.listAgentIds.mockReturnValue(["main", "writer"]); @@ -432,22 +465,11 @@ describe("models.authStatus", () => { const freshMain = createOptions({ agentId: "main" }); await handler(freshMain); - expect(mocks.ensureAuthProfileStore).toHaveBeenNthCalledWith( - 1, - "/tmp/agent", - expect.any(Object), - ); - expect(mocks.ensureAuthProfileStore).toHaveBeenNthCalledWith( - 2, - "/tmp/agent-writer", - expect.any(Object), - ); - expect(mocks.ensureAuthProfileStore).toHaveBeenNthCalledWith( - 3, - "/tmp/agent", - expect.any(Object), - ); - expect(mocks.ensureAuthProfileStore).toHaveBeenCalledTimes(3); + expect(mocks.readPreparedCatalog).toHaveBeenNthCalledWith(1, expect.anything(), "main"); + expect(mocks.readPreparedCatalog).toHaveBeenNthCalledWith(2, expect.anything(), "writer"); + expect(mocks.readPreparedCatalog).toHaveBeenNthCalledWith(3, expect.anything(), "main"); + expect(mocks.readPreparedCatalog).toHaveBeenCalledTimes(3); + expect(mocks.loadDeferredCatalog).not.toHaveBeenCalled(); expect(firstRespondCall(freshMain)?.[3]).toBeUndefined(); }); @@ -462,13 +484,15 @@ describe("models.authStatus", () => { await handler(createOptions({ refresh: true })); expect(mocks.getRuntimeConfig).toHaveBeenCalledTimes(2); + expect(mocks.readPreparedCatalog).not.toHaveBeenCalled(); + expect(mocks.loadDeferredCatalog).toHaveBeenCalledOnce(); expect(mocks.buildAuthHealthSummary).toHaveBeenCalledWith( expect.objectContaining({ cfg: after }), ); }); it("returns a serialisable snapshot on first call", async () => { - mocks.ensureAuthProfileStore.mockReturnValue({ + setPreparedAuthStore({ version: 1, profiles: { "openai:default": { @@ -509,9 +533,52 @@ describe("models.authStatus", () => { expect(result.providers[0]?.profiles[0]?.logoutSupported).toBe(true); }); + it("projects provider capabilities from the published lifecycle metadata", async () => { + const plugins = [ + { + id: "provider-auth", + origin: "bundled", + providerAuthAliases: { "openai-legacy": "openai" }, + providerAuthChoices: [ + { + provider: "openai-legacy", + method: "api-key", + choiceId: "openai-api-key", + choiceLabel: "OpenAI API key", + appGuidedSecret: true, + }, + { + provider: "openai", + method: "oauth", + choiceId: "openai-oauth", + choiceLabel: "OpenAI OAuth", + }, + { + provider: "github-copilot", + method: "oauth", + choiceId: "github-copilot-oauth", + choiceLabel: "GitHub Copilot OAuth", + }, + ], + }, + ]; + setPreparedMetadataSnapshot({ + index: { plugins: [] }, + manifestRegistry: { plugins }, + plugins, + }); + + const result = await readAuthStatus(); + + expect(result.providerCapabilities).toEqual([ + { provider: "github-copilot", apiKeySupported: false, quickApiKeySetup: false }, + { provider: "openai", apiKeySupported: true, quickApiKeySetup: true }, + ]); + }); + it("does not offer logout for runtime external CLI profiles", async () => { const health = createOpenAiCodexOauthHealthSummary(); - mocks.ensureAuthProfileStore.mockReturnValue({ + setPreparedAuthStore({ version: 1, profiles: {}, runtimeExternalProfileIds: ["openai:default"], @@ -540,7 +607,7 @@ describe("models.authStatus", () => { }, }, }); - mocks.ensureAuthProfileStore.mockReturnValue({ + setPreparedAuthStore({ version: 1, profiles: { [profileId]: { type: "token", provider: "openrouter", token: "placeholder" }, @@ -736,7 +803,7 @@ describe("models.authStatus", () => { providers: { anthropic: Object.fromEntries([["apiKey", profileId]]) }, }, }); - mocks.ensureAuthProfileStore.mockReturnValue({ + setPreparedAuthStore({ version: 1, profiles: { [profileId]: { @@ -809,20 +876,16 @@ describe("models.authStatus", () => { await handler(createOptions({ refresh: true })); expect(mocks.buildAuthHealthSummary).toHaveBeenCalledTimes(2); expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1); - expect(mocks.clearRuntimeAuthProfileStoreSnapshots).toHaveBeenCalledTimes(1); - const clearOrder = mocks.clearRuntimeAuthProfileStoreSnapshots.mock.invocationCallOrder[0]; - const refreshReadOrder = mocks.ensureAuthProfileStore.mock.invocationCallOrder.at(-1); - expect(clearOrder).toBeLessThan(refreshReadOrder ?? 0); + expect(mocks.loadDeferredCatalog).toHaveBeenCalledTimes(1); }); - it("keeps refreshed secrets runtime snapshots on explicit refresh", async () => { + it("refreshes the transient owner after secrets runtime refresh", async () => { mocks.refreshActiveProviderAuthRuntimeSnapshot.mockResolvedValueOnce(true); await handler(createOptions({ refresh: true })); expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1); - expect(mocks.clearRuntimeAuthProfileStoreSnapshots).not.toHaveBeenCalled(); - expect(mocks.ensureAuthProfileStore).toHaveBeenCalledTimes(1); + expect(mocks.loadDeferredCatalog).toHaveBeenCalledTimes(1); }); it("keeps last-good secrets runtime snapshots when explicit refresh fails", async () => { @@ -833,8 +896,7 @@ describe("models.authStatus", () => { await handler(createOptions({ refresh: true })); expect(mocks.refreshActiveProviderAuthRuntimeSnapshot).toHaveBeenCalledTimes(1); - expect(mocks.clearRuntimeAuthProfileStoreSnapshots).not.toHaveBeenCalled(); - expect(mocks.ensureAuthProfileStore).toHaveBeenCalledTimes(1); + expect(mocks.loadDeferredCatalog).toHaveBeenCalledTimes(1); }); it("invalidateModelAuthStatusCache() preserves fresh auth reads", async () => { @@ -1125,7 +1187,7 @@ describe("models.authStatus", () => { it("does not reuse usage after credentials rotate within the same provider", async () => { mocks.buildAuthHealthSummary.mockReturnValue(createOpenAiCodexOauthHealthSummary()); - mocks.ensureAuthProfileStore.mockReturnValue({ + setPreparedAuthStore({ version: 1, profiles: { "openai:default": { @@ -1154,7 +1216,7 @@ describe("models.authStatus", () => { expect(warmed.providers[0]?.usage?.windows[0]?.usedPercent).toBe(10); }); - mocks.ensureAuthProfileStore.mockReturnValue({ + setPreparedAuthStore({ version: 1, profiles: { "openai:default": { @@ -1224,7 +1286,7 @@ describe("models.authStatus", () => { expires: 1_000_000, }, }; - mocks.ensureAuthProfileStore.mockReturnValue({ + setPreparedAuthStore({ version: 1, profiles, lastGood: { openai: "openai:first" }, @@ -1246,7 +1308,7 @@ describe("models.authStatus", () => { expect(warmed.providers[0]?.usage?.windows[0]?.usedPercent).toBe(10); }); - mocks.ensureAuthProfileStore.mockReturnValue({ + setPreparedAuthStore({ version: 1, profiles, lastGood: { openai: "openai:second" }, @@ -1279,24 +1341,19 @@ describe("models.authStatus", () => { }, }); - await handler(createOptions()); + await handler(createOptions({ refresh: true })); - const externalCli = firstExternalCliAuthOption(); - expect(externalCli.mode).toBe("scoped"); - expect(externalCli.allowKeychainPrompt).toBe(false); - requireRecord(externalCli.config); - expect(externalCli.providerIds).toContain("opencode-go"); - expect(externalCli.providerIds).not.toContain("claude-cli"); - expect(externalCli.profileIds).toEqual(["opencode-go:default"]); + const authScope = firstDeferredAuthScope(); + expect(authScope.providerIds).toContain("opencode-go"); + expect(authScope.providerIds).not.toContain("claude-cli"); + expect(authScope.profileIds).toEqual(["opencode-go:default"]); }); it("disables external CLI auth overlays when config has no provider signal", async () => { - await handler(createOptions()); + await handler(createOptions({ refresh: true })); - const externalCli = firstExternalCliAuthOption(); - expect(externalCli.mode).toBe("none"); - expect(externalCli.allowKeychainPrompt).toBe(false); - requireRecord(externalCli.config); + const authScope = firstDeferredAuthScope(); + expect(authScope).toEqual({ providerIds: [] }); }); it("still returns providers when usage fetch fails", async () => { diff --git a/src/gateway/server-methods/models-auth-status.ts b/src/gateway/server-methods/models-auth-status.ts index 46dcc6a9387a..10592f03475a 100644 --- a/src/gateway/server-methods/models-auth-status.ts +++ b/src/gateway/server-methods/models-auth-status.ts @@ -16,8 +16,6 @@ import { } from "../../agents/auth-health.js"; import { type AuthProfileStore, - clearRuntimeAuthProfileStoreSnapshots, - ensureAuthProfileStore, ensureAuthProfileStoreWithoutExternalProfiles, externalCliDiscoveryForConfigStatus, listProfilesForProvider, @@ -49,8 +47,11 @@ import { coerceSecretRef, hasConfiguredSecretInput } from "../../config/types.se import { providerUsageLabel, resolveUsageProviderId } from "../../infra/provider-usage.shared.js"; import type { UsageProviderId } from "../../infra/provider-usage.types.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; +import { resolveManifestProviderAuthChoices } from "../../plugins/provider-auth-choices.js"; import { refreshActiveProviderAuthRuntimeSnapshot } from "../../secrets/runtime.js"; +import { supportsSetupManualSecret } from "../../system-agent/setup-inference-auth-options.js"; import { abortChatRunsForProvider, type ChatAbortOps } from "../chat-abort.js"; +import { loadDeferredCatalog, readPreparedCatalog } from "../server-model-catalog-auth.js"; import { formatForLog } from "../ws-log.js"; import { modelAuthAgentScopeError, resolveModelAuthAgentScope } from "./model-auth-agent-scope.js"; import { @@ -64,6 +65,7 @@ import type { ModelAuthLogoutResult, ModelAuthStatusProvider, ModelAuthStatusResult, + ModelProviderCapability, } from "./models-auth-status.types.js"; import type { GatewayRequestContext, GatewayRequestHandlers } from "./types.js"; @@ -73,11 +75,63 @@ export type { ModelAuthStatusProfile, ModelAuthStatusProvider, ModelAuthStatusResult, + ModelProviderCapability, } from "./models-auth-status.types.js"; const log = createSubsystemLogger("models-auth-status"); const apiKeyUsageStatusProviders = new Set(["clawrouter", "deepseek"]); +function buildProviderCapabilities(params: { + config: OpenClawConfig; + workspaceDir: string; + metadataSnapshot: NonNullable< + Awaited> + >["metadataSnapshot"]; +}): ModelProviderCapability[] { + const capabilities = new Map(); + for (const choice of resolveManifestProviderAuthChoices({ + config: params.config, + workspaceDir: params.workspaceDir, + includeUntrustedWorkspacePlugins: false, + metadataSnapshot: params.metadataSnapshot, + })) { + const provider = resolveProviderIdForAuth(choice.providerId, { + config: params.config, + workspaceDir: params.workspaceDir, + includeUntrustedWorkspacePlugins: false, + metadataSnapshot: params.metadataSnapshot, + }); + if (!provider) { + continue; + } + const current = capabilities.get(provider); + const apiKeySupported = choice.methodId === "api-key"; + const quickApiKeySetup = apiKeySupported && supportsSetupManualSecret(choice); + capabilities.set(provider, { + provider, + apiKeySupported: current?.apiKeySupported === true || apiKeySupported, + quickApiKeySetup: current?.quickApiKeySetup === true || quickApiKeySetup, + }); + } + return [...capabilities.values()].toSorted((a, b) => a.provider.localeCompare(b.provider)); +} + +function resolveAuthRefreshScope(cfg: OpenClawConfig): { + providerIds: string[]; + profileIds?: string[]; +} { + const discovery = externalCliDiscoveryForConfigStatus({ cfg }); + if (discovery.mode !== "scoped") { + return { providerIds: [] }; + } + const providerIds = [...(discovery.providerIds ?? [])]; + const profileIds = [...(discovery.profileIds ?? [])]; + return { + providerIds, + ...(profileIds.length > 0 ? { profileIds } : {}), + }; +} + /** * Invalidate auxiliary usage and prepared provider-auth state after an auth * mutation. Auth health itself is rebuilt on every request; only outbound @@ -99,16 +153,10 @@ async function refreshModelAuthStatusRuntimeState(): Promise { // still uses invalidateModelAuthStatusCache() and clears usage immediately. clearCurrentProviderAuthState(); try { - if (await refreshActiveProviderAuthRuntimeSnapshot()) { - return; - } + await refreshActiveProviderAuthRuntimeSnapshot(); } catch (err) { log.warn(`runtime auth snapshot refresh before auth status failed: ${formatForLog(err)}`); - return; } - // Explicit status refresh follows CLI/doctor repairs. If no secrets runtime is - // active, drop runtime auth snapshots so the next status read observes disk. - clearRuntimeAuthProfileStoreSnapshots(); } function readProviderParam(params: Record): string | null { @@ -578,12 +626,18 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = { return; } } - const { agentId, agentDir } = scope; - // Use the external-profile-aware store for status reads so the dashboard - // reflects CLI-discovered credentials without persisting them here. - const store = ensureAuthProfileStore(agentDir, { - externalCli: externalCliDiscoveryForConfigStatus({ cfg }), - }); + const preparedSnapshot = refreshRequested + ? await loadDeferredCatalog(context, scope.agentId, { + readOnly: true, + authScope: resolveAuthRefreshScope(cfg), + refreshAuth: true, + }) + : await readPreparedCatalog(context, scope.agentId); + if (!preparedSnapshot) { + throw new Error(`prepared model auth owner is unavailable (${scope.agentId})`); + } + cfg = preparedSnapshot.config; + const { agentId, agentDir, authStore: store, workspaceDir } = preparedSnapshot; const apiKeys = resolveProviderApiKeys(cfg, store); const configured = resolveConfiguredProviders(cfg, apiKeys); const statusProviderIds = new Set(configured.providers); @@ -601,6 +655,11 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = { cfg, providers: statusProviderIds.size > 0 ? [...statusProviderIds] : undefined, allowKeychainPrompt: false, + authAliasLookupParams: { + workspaceDir, + metadataSnapshot: preparedSnapshot.metadataSnapshot, + includeUntrustedWorkspacePlugins: false, + }, }); // Usage queries usually need refreshable credentials. Keep API-key status @@ -657,7 +716,12 @@ export const modelsAuthStatusHandlers: GatewayRequestHandlers = { configBoundProfileIds, ), ); - const result: ModelAuthStatusResult = { ts: now, providers }; + const providerCapabilities = buildProviderCapabilities({ + config: cfg, + workspaceDir, + metadataSnapshot: preparedSnapshot.metadataSnapshot, + }); + const result: ModelAuthStatusResult = { ts: now, providers, providerCapabilities }; respond(true, result, undefined); } catch (err) { respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, formatForLog(err))); diff --git a/src/gateway/server-methods/models-auth-status.types.ts b/src/gateway/server-methods/models-auth-status.types.ts index 1d10dc7a94a3..c2a535ef914f 100644 --- a/src/gateway/server-methods/models-auth-status.types.ts +++ b/src/gateway/server-methods/models-auth-status.types.ts @@ -47,10 +47,18 @@ export type ModelAuthStatusProvider = { }; }; +export type ModelProviderCapability = { + provider: string; + apiKeySupported: boolean; + quickApiKeySetup: boolean; +}; + export type ModelAuthStatusResult = { /** Snapshot build time, ms since epoch. 0 = never loaded (UI fallback sentinel). */ ts: number; providers: ModelAuthStatusProvider[]; + /** Process-stable provider setup capabilities from the active plugin generation. */ + providerCapabilities?: ModelProviderCapability[]; }; export type ModelAuthLogoutResult = { diff --git a/src/gateway/server-methods/models-list-result.ts b/src/gateway/server-methods/models-list-result.ts index 03dbc82017d1..ca1afd5bb36e 100644 --- a/src/gateway/server-methods/models-list-result.ts +++ b/src/gateway/server-methods/models-list-result.ts @@ -511,6 +511,8 @@ export async function buildModelsListResult( const initialConfig = params.context.getRuntimeConfig(); const initialAgentId = normalizeAgentId(params.agentId ?? resolveDefaultAgentId(initialConfig)); const view = resolveModelsListView(params.params); + const preparedOnly = params.params.preparedOnly === true; + const refresh = params.params.refresh === true; const preloadedCatalog = params.preloadedCatalog?.agentId === initialAgentId && preparedModelRuntimeConfigsMatch(params.preloadedCatalog.config, initialConfig) @@ -532,6 +534,8 @@ export async function buildModelsListResult( cfg: initialConfig, agentId: initialAgentId, view, + preparedOnly, + refresh, loadCatalog: async (loadParams) => { loadedReadOnly = loadParams.readOnly ?? true; // A read-only preload cannot satisfy a full-discovery request. Reuse it only when the @@ -546,7 +550,11 @@ export async function buildModelsListResult( if (params.preloadedOnly) { return { entries: [], routeVariants: [] }; } - loadedSnapshot = await loadDeferredCatalog(params.context, initialAgentId, loadedReadOnly); + loadedSnapshot = await loadDeferredCatalog(params.context, initialAgentId, { + readOnly: loadedReadOnly, + refreshAuth: refresh && loadedReadOnly, + refreshFullCatalog: loadParams.refresh === true, + }); return loadedSnapshot; }, onTimeout: handleCatalogTimeout, @@ -554,6 +562,7 @@ export async function buildModelsListResult( if ( loadedSnapshot && loadedReadOnly && + !preparedOnly && modelCatalogBrowseRequiresFullDiscovery({ cfg: loadedSnapshot.config, agentId: loadedSnapshot.agentId, @@ -567,8 +576,13 @@ export async function buildModelsListResult( cfg: loadedSnapshot.config, agentId: escalationAgentId, view, + refresh, loadCatalog: async ({ readOnly }) => { - fullSnapshot = await loadDeferredCatalog(params.context, escalationAgentId, readOnly); + fullSnapshot = await loadDeferredCatalog(params.context, escalationAgentId, { + readOnly, + refreshAuth: refresh && readOnly, + refreshFullCatalog: refresh, + }); return fullSnapshot; }, timeoutFullDiscovery: true, diff --git a/src/gateway/server-methods/models.ts b/src/gateway/server-methods/models.ts index 144a5d085cb7..0c31bdca5f19 100644 --- a/src/gateway/server-methods/models.ts +++ b/src/gateway/server-methods/models.ts @@ -1,6 +1,5 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; -// Models gateway methods expose model catalog browse results without triggering -// auth probes or fresh provider discovery on each request. +// Models gateway methods expose prepared, cached, and explicitly refreshed catalog views. import { validateModelsListParams } from "../../../packages/gateway-protocol/src/index.js"; import { resolveAgentIdOrRespondError } from "./agent-id-shared.js"; import { buildModelsListResult } from "./models-list-result.js"; @@ -9,9 +8,7 @@ import { assertValidParams } from "./validation.js"; export { buildModelsListResult }; -// The gateway model list is a browse API, not an auth probe. It reuses the -// current runtime catalog snapshot and applies visibility rules without doing -// extra runtime discovery on each request. +// Automatic clients opt into preparedOnly; omitted mode preserves shipped wildcard discovery. export const modelsHandlers: GatewayRequestHandlers = { "models.list": async ({ params, respond, context }) => { if (!assertValidParams(params, validateModelsListParams, "models.list", respond)) { diff --git a/src/gateway/server-model-catalog-auth.ts b/src/gateway/server-model-catalog-auth.ts index f9a9d4c07f37..9b946ca2fb55 100644 --- a/src/gateway/server-model-catalog-auth.ts +++ b/src/gateway/server-model-catalog-auth.ts @@ -1,5 +1,6 @@ import type { RuntimeAuthMaterialization } from "../agents/auth-profiles/runtime-materializations.js"; import type { ResolvedPublishedModelCatalogOwner } from "../agents/prepared-model-catalog.types.js"; +import type { PreparedModelRuntimeAuthScope } from "../agents/prepared-model-runtime-auth.js"; import type { GatewayRequestContext } from "./server-methods/shared-types.js"; import type { GatewayModelCatalogSnapshot } from "./server-model-catalog.types.js"; @@ -11,7 +12,10 @@ export type PreparedGatewayModelCatalogSnapshot = GatewayModelCatalogSnapshot & type GatewayModelCatalogReadParams = { agentId?: string; agentDir?: string; + authScope?: PreparedModelRuntimeAuthScope; readOnly?: boolean; + refreshAuth?: boolean; + refreshFullCatalog?: boolean; workspaceDir?: string; }; @@ -50,9 +54,12 @@ function requirePrivateAccess( export async function loadDeferredCatalog( context: Pick, agentId: string, - readOnly: boolean, + options: Pick< + GatewayModelCatalogReadParams, + "authScope" | "readOnly" | "refreshAuth" | "refreshFullCatalog" + >, ): Promise { - return await requirePrivateAccess(context).loadDeferred({ agentId, readOnly }); + return await requirePrivateAccess(context).loadDeferred({ agentId, ...options }); } export async function readPreparedCatalog( diff --git a/src/gateway/server-model-catalog.test.ts b/src/gateway/server-model-catalog.test.ts index d84047fccceb..cfbd1561f5d6 100644 --- a/src/gateway/server-model-catalog.test.ts +++ b/src/gateway/server-model-catalog.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import type { ModelCatalogSnapshot } from "../agents/model-catalog.types.js"; import type { PublishedModelCatalogOwnerCandidate } from "../agents/prepared-model-catalog.types.js"; import { setPreparedModelRuntimeAuthLoader } from "../agents/prepared-model-runtime-auth.js"; +import { PreparedModelRuntimePublicationSupersededError } from "../agents/prepared-model-runtime.errors.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { loadDeferredCatalog, @@ -134,24 +135,34 @@ describe("gateway prepared model catalog", () => { }), ); registerGatewayModelCatalogPrivateAccess(publicLoader, { - loadDeferred: () => + loadDeferred: (params) => loadPreparedGatewayModelCatalogSnapshot({ + ...params, getConfig: () => config, loadPublishedPreparedModelCatalogOwnerSnapshot, - refreshAuth: true, }), readPrepared: async () => undefined, }); const loaded = await loadDeferredCatalog( { loadGatewayModelCatalogSnapshot: publicLoader }, "main", - true, + { + readOnly: true, + authScope: { + providerIds: ["openai"], + profileIds: ["openai:refreshed"], + }, + refreshAuth: true, + }, ); expect(loaded.authStore).toEqual( expect.objectContaining({ profiles: { "openai:refreshed": expect.any(Object) } }), ); expect(loaded.authModes).toEqual({ openai: "api_key" }); - expect(loadAuth).toHaveBeenCalledWith(["openai"]); + expect(loadAuth).toHaveBeenCalledWith({ + providerIds: ["openai"], + profileIds: ["openai:refreshed"], + }); }); it("removes stale prepared auth modes when deferred auth observes logout", async () => { @@ -172,24 +183,88 @@ describe("gateway prepared model catalog", () => { }), ); registerGatewayModelCatalogPrivateAccess(publicLoader, { - loadDeferred: () => + loadDeferred: (params) => loadPreparedGatewayModelCatalogSnapshot({ + ...params, getConfig: () => config, loadPublishedPreparedModelCatalogOwnerSnapshot: async () => candidate, - refreshAuth: true, }), readPrepared: async () => undefined, }); const loaded = await loadDeferredCatalog( { loadGatewayModelCatalogSnapshot: publicLoader }, "main", - true, + { readOnly: true, refreshAuth: true }, ); expect(loaded.authStore?.profiles).toEqual({}); expect(loaded.authModes).toEqual({}); }); + it("retries the whole owner projection when deferred auth supersedes its generation", async () => { + const staleConfig = ownerConfig("main", { logging: { level: "info" } }); + const currentConfig = ownerConfig("main", { logging: { level: "debug" } }); + const staleCatalog: ModelCatalogSnapshot = { + entries: [{ provider: "openai", id: "stale", name: "Stale" }], + routeVariants: [], + }; + const currentCatalog: ModelCatalogSnapshot = { + entries: [{ provider: "openai", id: "current", name: "Current" }], + routeVariants: [], + }; + const stale = { + ...ownerSnapshot(staleConfig, staleCatalog), + authModes: { openai: "oauth" as const }, + authStore: { + version: 1 as const, + profiles: { + "openai:stale": { + type: "token" as const, + provider: "openai", + token: "stale-token-not-real", + }, + }, + }, + }; + const current = { + ...ownerSnapshot(currentConfig, currentCatalog), + authModes: { openai: "api_key" as const }, + authStore: { + version: 1 as const, + profiles: { + "openai:current": { + type: "api_key" as const, + provider: "openai", + key: "current-key-not-real", + }, + }, + }, + }; + setPreparedModelRuntimeAuthLoader(stale, async () => { + throw new PreparedModelRuntimePublicationSupersededError("superseded"); + }); + const loadPublishedPreparedModelCatalogOwnerSnapshot = vi + .fn() + .mockResolvedValueOnce(stale) + .mockResolvedValueOnce(current); + + await expect( + loadPreparedGatewayModelCatalogSnapshot({ + getConfig: () => staleConfig, + loadPublishedPreparedModelCatalogOwnerSnapshot, + refreshAuth: true, + }), + ).resolves.toMatchObject({ + config: currentConfig, + entries: currentCatalog.entries, + authModes: { openai: "api_key" }, + authStore: { + profiles: { "openai:current": expect.any(Object) }, + }, + }); + expect(loadPublishedPreparedModelCatalogOwnerSnapshot).toHaveBeenCalledTimes(2); + }); + it("rejects an ambiguous owner without an authoritative agent identity", async () => { const config = { agents: { @@ -248,11 +323,13 @@ describe("gateway prepared model catalog", () => { getConfig: () => config, loadPublishedPreparedModelCatalogOwnerSnapshot, readOnly: false, + refreshFullCatalog: true, }), ).resolves.toMatchObject(snapshot); expect(loadPublishedPreparedModelCatalogOwnerSnapshot).toHaveBeenCalledWith({ config, readOnly: false, + refreshFullCatalog: true, }); }); diff --git a/src/gateway/server-model-catalog.ts b/src/gateway/server-model-catalog.ts index 46c51607ecc1..f545c7ec13b8 100644 --- a/src/gateway/server-model-catalog.ts +++ b/src/gateway/server-model-catalog.ts @@ -6,7 +6,9 @@ import type { import { getPreparedModelRuntimeAuthMaterializations, loadPreparedModelRuntimeAuth, + type PreparedModelRuntimeAuthScope, } from "../agents/prepared-model-runtime-auth.js"; +import { PreparedModelRuntimePublicationSupersededError } from "../agents/prepared-model-runtime.errors.js"; // Gateway catalog reads use the atomic prepared runtime generation. import { getRuntimeConfig } from "../config/io.js"; import type { PreparedGatewayModelCatalogSnapshot } from "./server-model-catalog-auth.js"; @@ -21,6 +23,7 @@ type LoadPublishedPreparedModelCatalogOwnerSnapshot = (params: { agentDir?: string; config: GatewayModelCatalogConfig; readOnly?: boolean; + refreshFullCatalog?: boolean; workspaceDir?: string; }) => Promise; type LoadGatewayModelCatalogParams = { @@ -29,9 +32,11 @@ type LoadGatewayModelCatalogParams = { getConfig?: () => GatewayModelCatalogConfig; loadPublishedPreparedModelCatalogOwnerSnapshot?: LoadPublishedPreparedModelCatalogOwnerSnapshot; readOnly?: boolean; + refreshFullCatalog?: boolean; workspaceDir?: string; }; type LoadPreparedGatewayModelCatalogParams = LoadGatewayModelCatalogParams & { + authScope?: PreparedModelRuntimeAuthScope; refreshAuth?: boolean; }; @@ -71,6 +76,7 @@ async function loadGatewayModelCatalogOwnerSnapshot( ...(params?.agentDir ? { agentDir: params.agentDir } : {}), config: (params?.getConfig ?? getRuntimeConfig)(), readOnly: params?.readOnly !== false, + ...(params?.refreshFullCatalog ? { refreshFullCatalog: true } : {}), ...(params?.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), }); const owner = resolvePublishedModelCatalogOwner(candidate); @@ -101,20 +107,34 @@ function projectGatewayModelCatalogSnapshot( export async function loadPreparedGatewayModelCatalogSnapshot( params?: LoadPreparedGatewayModelCatalogParams, ): Promise { - const { candidate, owner } = await loadGatewayModelCatalogOwnerSnapshot(params); - const refreshedAuth = params?.refreshAuth - ? await loadPreparedModelRuntimeAuth( - candidate, - owner.modelCatalog.entries.map((entry) => entry.provider), - ).catch(() => undefined) - : undefined; - return { - ...projectGatewayModelCatalogSnapshot(owner), - authModes: refreshedAuth?.authModes ?? owner.authModes, - authStore: refreshedAuth?.authStore ?? owner.authStore, - metadataSnapshot: owner.metadataSnapshot, - authMaterializations: owner.authMaterializations, - }; + for (;;) { + const { candidate, owner } = await loadGatewayModelCatalogOwnerSnapshot(params); + let refreshedAuth: Awaited>; + try { + refreshedAuth = params?.refreshAuth + ? await loadPreparedModelRuntimeAuth( + candidate, + params.authScope ?? { + providerIds: owner.modelCatalog.entries.map((entry) => entry.provider), + }, + ) + : undefined; + } catch (error) { + if (error instanceof PreparedModelRuntimePublicationSupersededError) { + // Supersession invalidates every captured owner fact. Reacquire the whole owner so + // replacement auth cannot be combined with stale catalog or metadata. + continue; + } + refreshedAuth = undefined; + } + return { + ...projectGatewayModelCatalogSnapshot(owner), + authModes: refreshedAuth?.authModes ?? owner.authModes, + authStore: refreshedAuth?.authStore ?? owner.authStore, + metadataSnapshot: owner.metadataSnapshot, + authMaterializations: owner.authMaterializations, + }; + } } export async function loadGatewayModelCatalogSnapshot( diff --git a/ui/src/components/model-picker.ts b/ui/src/components/model-picker.ts index 56205b8b5b26..83c7dacafb68 100644 --- a/ui/src/components/model-picker.ts +++ b/ui/src/components/model-picker.ts @@ -27,6 +27,7 @@ type ModelPickerParams = { invalid?: boolean; describedBy?: string; }; + onOpen?: () => void; onChange: (value: string) => void; }; @@ -52,6 +53,7 @@ export function renderModelPicker(params: ModelPickerParams) { title: params.title, placement: params.placement, className: `model-picker__select ${params.className ?? ""}`, + onOpen: params.onOpen, renderLeading: (option) => option.provider ? renderProviderBrandIcon(option.provider, { className: "model-picker__provider-icon" }) diff --git a/ui/src/components/select-picker.ts b/ui/src/components/select-picker.ts index 7eab8c93ed5e..2f28a17dc53d 100644 --- a/ui/src/components/select-picker.ts +++ b/ui/src/components/select-picker.ts @@ -17,6 +17,7 @@ export type PickerParams